From 0b578c1cbfec68b050c9e088bf326ab44cba147d Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 24 Jul 2026 13:24:54 +0200 Subject: [PATCH 001/111] trackref: use autodoc (#7771) --- docs/topics/leaks.rst | 25 ++++--------------------- scrapy/utils/trackref.py | 18 ++++++++++++++---- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index d4577601d..0a8b146d2 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -187,30 +187,13 @@ scrapy.utils.trackref module Here are the functions available in the :mod:`~scrapy.utils.trackref` module. -.. class:: object_ref +.. autoclass:: object_ref - Inherit from this class if you want to track live - instances with the ``trackref`` module. +.. autofunction:: print_live_refs(ignore=NoneType) -.. function:: print_live_refs(ignore=NoneType) +.. autofunction:: get_oldest - Print a report of live references, grouped by class name. - - :param ignore: if given, all objects from the specified class (or tuple of - classes) will be ignored. - :type ignore: type or tuple - -.. function:: get_oldest(class_name) - - Return the oldest object alive with the given class name, or ``None`` if - none is found. Use :func:`print_live_refs` first to get a list of all - tracked live objects per class name. - -.. function:: iter_all(class_name) - - Return an iterator over all objects alive with the given class name. Use - :func:`print_live_refs` first to get a list of all tracked live objects - per class name. +.. autofunction:: iter_all .. skip: end diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 22f9eadd0..0bbe68def 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -34,7 +34,8 @@ live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict( class object_ref: - """Inherit from this class to a keep a record of live instances""" + """Inherit from this class if you want to track live instances with the + ``trackref`` module.""" __slots__ = () @@ -60,12 +61,19 @@ def format_live_refs(ignore: Any = NoneType) -> str: def print_live_refs(*a: Any, **kw: Any) -> None: - """Print tracked objects""" + """Print a report of live references, grouped by class name. + + :param ignore: if given, all objects from the specified class (or tuple of + classes) will be ignored. + :type ignore: type or tuple + """ print(format_live_refs(*a, **kw)) def get_oldest(class_name: str) -> Any: - """Get the oldest object for a specific class name""" + """Return the oldest object alive with the given class name, or ``None`` if + none is found. Use :func:`print_live_refs` first to get a list of all + tracked live objects per class name.""" for cls, wdict in live_refs.items(): if cls.__name__ == class_name: if not wdict: @@ -75,7 +83,9 @@ def get_oldest(class_name: str) -> Any: def iter_all(class_name: str) -> Iterable[Any]: - """Iterate over all objects of the same class by its class name""" + """Return an iterator over all objects alive with the given class name. Use + :func:`print_live_refs` first to get a list of all tracked live objects per + class name.""" for cls, wdict in live_refs.items(): if cls.__name__ == class_name: return wdict.keys() From 41bb09741a5d24ccae8687fed990c41219d2f0d5 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 24 Jul 2026 15:35:54 +0200 Subject: [PATCH 002/111] Use autodoc for contracts (#7775) --- docs/topics/contracts.rst | 60 +++++-------------------------- scrapy/contracts/__init__.py | 18 +++++++++- scrapy/contracts/default.py | 68 ++++++++++++++++++++++++++---------- 3 files changed, 76 insertions(+), 70 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 61aef4bbb..df67bee02 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -30,43 +30,15 @@ You can use the following contracts: .. module:: scrapy.contracts.default -.. class:: UrlContract +.. autoclass:: UrlContract - This contract (``@url``) sets the sample URL used when checking other - contract conditions for this spider. This contract is mandatory. All - callbacks lacking this contract are ignored when running the checks:: +.. autoclass:: CallbackKeywordArgumentsContract - @url url +.. autoclass:: MetadataContract -.. class:: CallbackKeywordArgumentsContract +.. autoclass:: ReturnsContract - This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` - attribute for the sample request. It must be a valid JSON dictionary. - :: - - @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} - -.. class:: MetadataContract - - This contract (``@meta``) sets the :attr:`meta ` - attribute for the sample request. It must be a valid JSON dictionary. - :: - - @meta {"arg1": "value1", "arg2": "value2", ...} - -.. class:: ReturnsContract - - This contract (``@returns``) sets lower and upper bounds for the items and - requests returned by the spider. The upper bound is optional:: - - @returns item(s)|request(s) [min [max]] - -.. class:: ScrapesContract - - This contract (``@scrapes``) checks that all the items returned by the - callback have the specified fields:: - - @scrapes field_1 field_2 ... +.. autoclass:: ScrapesContract Use the :command:`check` command to run the contract checks. @@ -89,30 +61,16 @@ override three methods: .. module:: scrapy.contracts -.. class:: Contract(method, *args) +.. autoclass:: Contract - :param method: callback function to which the contract is associated - :type method: collections.abc.Callable + .. automethod:: adjust_request_args - :param args: list of arguments passed into the docstring (whitespace - separated) - :type args: list - - .. method:: Contract.adjust_request_args(args) - - This receives a ``dict`` as an argument containing default arguments - for request object. :class:`~scrapy.Request` is used by default, - but this can be changed with the ``request_cls`` attribute. - If multiple contracts in chain have this attribute defined, the last one is used. - - Must return the same or a modified version of it. - - .. method:: Contract.pre_process(response) + .. method:: pre_process(response) This allows hooking in various checks on the response received from the sample request, before it's being passed to the callback. - .. method:: Contract.post_process(output) + .. method:: post_process(output) This allows processing the output of the callback. Iterators are converted to lists before being passed to this hook. diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index ebbdf1b98..dbdbbd456 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -22,7 +22,16 @@ if TYPE_CHECKING: class Contract: - """Abstract class for contracts""" + """Base class for :ref:`custom contracts `. + + *method* is the callback function to which the contract is associated. + + *args* is the list of arguments passed into the docstring, separated by + whitespace. + + Subclasses may override :meth:`adjust_request_args`, and define a + ``pre_process`` method or a ``post_process`` method, or both. + """ request_cls: type[Request] | None = None name: str @@ -90,6 +99,13 @@ class Contract: return request def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: + """Receive a ``dict`` with the default arguments for the sample request + and return it, either unmodified or with changes. + + :class:`~scrapy.Request` is used by default, but this can be changed + with the ``request_cls`` attribute. If multiple contracts in the chain + define this attribute, the last one is used. + """ return args diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 9b42ca36f..e2e27165a 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -15,8 +15,15 @@ if TYPE_CHECKING: # contracts class UrlContract(Contract): - """Contract to set the url of the request (mandatory) - @url http://scrapy.org + """Sets (``@url``) the sample URL used when checking the other contract + conditions of a callback. + + This contract is mandatory: callbacks lacking it are ignored when running + the checks. + + .. code-block:: none + + @url url """ name = "url" @@ -27,10 +34,14 @@ class UrlContract(Contract): class CallbackKeywordArgumentsContract(Contract): - """Contract to set the keyword arguments for the request. - The value should be a JSON-encoded dictionary, e.g.: + """Sets (``@cb_kwargs``) the :attr:`cb_kwargs ` + attribute of the sample request. - @cb_kwargs {"arg1": "some value"} + Its value must be a valid JSON dictionary. + + .. code-block:: none + + @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} """ name = "cb_kwargs" @@ -41,10 +52,14 @@ class CallbackKeywordArgumentsContract(Contract): class MetadataContract(Contract): - """Contract to set metadata arguments for the request. - The value should be JSON-encoded dictionary, e.g.: + """Sets (``@meta``) the :attr:`meta ` attribute of the + sample request. - @meta {"arg1": "some value"} + Its value must be a valid JSON dictionary. + + .. code-block:: none + + @meta {"arg1": "value1", "arg2": "value2", ...} """ name = "meta" @@ -55,16 +70,29 @@ class MetadataContract(Contract): class ReturnsContract(Contract): - """Contract to check the output of a callback + """Sets (``@returns``) lower and upper bounds for the items and requests + returned by a callback. - general form: - @returns request(s)/item(s) [min=1 [max]] + The upper bound is optional: - e.g.: - @returns request - @returns request 2 - @returns request 2 10 - @returns request 0 10 + .. code-block:: none + + @returns item(s)|request(s) [min [max]] + + For example: + + .. code-block:: none + + @returns request + @returns request 2 + @returns request 2 10 + @returns request 0 10 + + Set both bounds to the same value to require an exact number: + + .. code-block:: none + + @returns request 2 2 """ name = "returns" @@ -115,8 +143,12 @@ class ReturnsContract(Contract): class ScrapesContract(Contract): - """Contract to check presence of fields in scraped items - @scrapes page_name page_body + """Checks (``@scrapes``) that all items returned by a callback have the + specified fields. + + .. code-block:: none + + @scrapes field_1 field_2 ... """ name = "scrapes" From 58ed9fdcccc9d8a3c278a0bd0af54ff77863b034 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 24 Jul 2026 17:08:58 +0200 Subject: [PATCH 003/111] Document urlparse_cached (#7777) --- docs/topics/request-response.rst | 2 ++ scrapy/utils/httpobj.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 7468f1b64..b83a04032 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -342,6 +342,8 @@ Other functions related to requests .. autofunction:: scrapy.utils.request.request_from_dict +.. autofunction:: scrapy.utils.httpobj.urlparse_cached + .. _topics-request-response-ref-request-callback-arguments: diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index 58b4539bf..5965e1f21 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -16,8 +16,15 @@ _urlparse_cache: WeakKeyDictionary[Request | Response, ParseResult] = ( def urlparse_cached(request_or_response: Request | Response) -> ParseResult: - """Return urlparse.urlparse caching the result, where the argument can be a - Request or Response object + """Return the result of parsing the URL of *request_or_response*, a + :class:`~scrapy.Request` or :class:`~scrapy.http.Response` object, with + :func:`urllib.parse.urlparse`. + + The result is cached, using a :class:`weakref.WeakKeyDictionary` keyed on + *request_or_response*, so that the URL of a given object is parsed only + once. Prefer this function over calling :func:`urllib.parse.urlparse` on + ``request_or_response.url`` directly when the same URL may be parsed more + than once. """ if request_or_response not in _urlparse_cache: _urlparse_cache[request_or_response] = urlparse(request_or_response.url) From e4ae4aad52145ba3e525be0d1ed90b008ee6fa6d Mon Sep 17 00:00:00 2001 From: SpiliosDmk <150211937+SpiliosDimakopoulos@users.noreply.github.com> Date: Sat, 25 Jul 2026 08:19:53 +0300 Subject: [PATCH 004/111] docs: fix missing 'of' in faq.rst (#7780) 'instead joining the strings' -> 'instead of joining the strings'. --- docs/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 8f2013581..dc909cf56 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -136,7 +136,7 @@ middleware with a :ref:`custom downloader middleware ` that requires less memory. For example: - If your domain names are similar enough, use your own regular expression - instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into + instead of joining the strings in :attr:`~scrapy.Spider.allowed_domains` into a complex regular expression. - If you can meet the installation requirements, use pyre2_ instead of From 96195e4a61209070845df40c3ba046a00f1e5714 Mon Sep 17 00:00:00 2001 From: SpiliosDmk <150211937+SpiliosDimakopoulos@users.noreply.github.com> Date: Sat, 25 Jul 2026 08:20:18 +0300 Subject: [PATCH 005/111] docs: fix a/an article before XML in faq.rst (#7779) 'a XML document' -> 'an XML document' (XML is pronounced with a vowel sound). --- docs/faq.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index dc909cf56..0446a6868 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -332,8 +332,8 @@ section of the site (which varies each time). In that case, the credentials to log in would be settings, while the url of the section to scrape would be a spider argument. -I'm scraping a XML document and my XPath selector doesn't return any items --------------------------------------------------------------------------- +I'm scraping an XML document and my XPath selector doesn't return any items +--------------------------------------------------------------------------- You may need to remove namespaces. See :ref:`removing-namespaces`. From e710b9c18e18f0a3fe104fbfc72d49c221dfe448 Mon Sep 17 00:00:00 2001 From: SpiliosDmk <150211937+SpiliosDimakopoulos@users.noreply.github.com> Date: Sat, 25 Jul 2026 08:20:30 +0300 Subject: [PATCH 006/111] core: fix verb tense in http2 stream comment (#7778) 'needs to be send' -> 'needs to be sent'. Comment-only, no functional change. --- scrapy/core/http2/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 4d072c555..c6226bbca 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -315,7 +315,7 @@ class Stream: 0, self.metadata["remaining_content_length"] ) - # End the stream if no more data needs to be send + # End the stream if no more data needs to be sent if self.metadata["remaining_content_length"] == 0: self._protocol.conn.end_stream(self.stream_id) From 13be37e4b1076643d2914bb891ba68dd84675d99 Mon Sep 17 00:00:00 2001 From: SpiliosDmk <150211937+SpiliosDimakopoulos@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:07:32 +0300 Subject: [PATCH 007/111] Add type hints to test_link.py and test_downloadermiddleware_stats.py (#7785) --- pyproject.toml | 3 --- tests/test_downloadermiddleware_stats.py | 25 +++++++++++++++--------- tests/test_link.py | 16 ++++++++------- tests/test_logstats.py | 11 +++++++---- 4 files changed, 32 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4220b12ed..5ed07a9ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -136,7 +136,6 @@ module = [ "tests.test_downloadermiddleware_redirect_metarefresh", "tests.test_downloadermiddleware_retry", "tests.test_downloadermiddleware_robotstxt", - "tests.test_downloadermiddleware_stats", "tests.test_downloaderslotssettings", "tests.test_dupefilters", "tests.test_engine_loop", @@ -154,11 +153,9 @@ module = [ "tests.test_http_response", "tests.test_http_response_text", "tests.test_item", - "tests.test_link", "tests.test_linkextractors", "tests.test_loader", "tests.test_logformatter", - "tests.test_logstats", "tests.test_mail", "tests.test_pipeline_crawl", "tests.test_pipeline_files", diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 5609360a7..b8fda25b4 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_size @@ -12,8 +14,9 @@ class MyException(Exception): class TestDownloaderStats: - def setup_method(self): + def setup_method(self) -> None: self.crawler = get_crawler(Spider) + assert self.crawler.stats is not None self.mw = DownloaderStats(self.crawler.stats) self.crawler.stats.open_spider() @@ -21,20 +24,21 @@ class TestDownloaderStats: self.req = Request("http://scrapytest.org") self.res = Response("http://scrapytest.org", status=400) - def assertStatsEqual(self, key, value): + def assertStatsEqual(self, key: str, value: object) -> None: + assert self.crawler.stats is not None assert self.crawler.stats.get_value(key) == value, str( self.crawler.stats.get_stats() ) - def test_process_request(self): + def test_process_request(self) -> None: self.mw.process_request(self.req) self.assertStatsEqual("downloader/request_count", 1) - def test_process_response(self): + def test_process_response(self) -> None: self.mw.process_response(self.req, self.res) self.assertStatsEqual("downloader/response_count", 1) - def test_process_exception(self): + def test_process_exception(self) -> None: self.mw.process_exception(self.req, MyException()) self.assertStatsEqual("downloader/exception_count", 1) self.assertStatsEqual( @@ -42,14 +46,17 @@ class TestDownloaderStats: 1, ) - def test_from_crawler_not_configured(self): + def test_from_crawler_not_configured(self) -> None: crawler = get_crawler(Spider, {"DOWNLOADER_STATS": False}) with pytest.raises(NotConfigured): DownloaderStats.from_crawler(crawler) - def teardown_method(self): + def teardown_method(self) -> None: + assert self.crawler.stats is not None self.crawler.stats.close_spider() -def test_get_header_size_non_list_value(): - assert get_header_size({"Content-Type": "text/html"}) == 0 +def test_get_header_size_non_list_value() -> None: + # Deliberately passing a non-list/tuple header value to make sure + # get_header_size() degrades gracefully instead of raising. + assert get_header_size({"Content-Type": "text/html"}) == 0 # type: ignore[dict-item] diff --git a/tests/test_link.py b/tests/test_link.py index 0eeffe12b..40c53edd3 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -1,18 +1,20 @@ +from __future__ import annotations + import pytest from scrapy.link import Link class TestLink: - def _assert_same_links(self, link1, link2): + def _assert_same_links(self, link1: Link, link2: Link) -> None: assert link1 == link2 assert hash(link1) == hash(link2) - def _assert_different_links(self, link1, link2): + def _assert_different_links(self, link1: Link, link2: Link) -> None: assert link1 != link2 assert hash(link1) != hash(link2) - def test_eq_and_hash(self): + def test_eq_and_hash(self) -> None: l1 = Link("http://www.example.com") l2 = Link("http://www.example.com/other") l3 = Link("http://www.example.com") @@ -45,17 +47,17 @@ class TestLink: self._assert_different_links(l7, l9) self._assert_different_links(l7, l10) - def test_repr(self): + def test_repr(self) -> None: l1 = Link( "http://www.example.com", text="test", fragment="something", nofollow=True ) l2 = eval(repr(l1)) self._assert_same_links(l1, l2) - def test_bytes_url(self): + def test_bytes_url(self) -> None: with pytest.raises(TypeError): - Link(b"http://www.example.com/\xc2\xa3") + Link(b"http://www.example.com/\xc2\xa3") # type: ignore[arg-type] - def test_eq_non_link(self): + def test_eq_non_link(self) -> None: url = "http://example.com" assert Link(url) != url diff --git a/tests/test_logstats.py b/tests/test_logstats.py index 370728e6a..213681ad1 100644 --- a/tests/test_logstats.py +++ b/tests/test_logstats.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from datetime import datetime import pytest @@ -9,16 +11,17 @@ from tests.utils.decorators import coroutine_test class TestLogStats: - def setup_method(self): + def setup_method(self) -> None: self.crawler = get_crawler(SimpleSpider) self.spider = self.crawler._create_spider("spidey") + assert self.crawler.stats is not None self.stats = self.crawler.stats self.stats.set_value("response_received_count", 4802) self.stats.set_value("item_scraped_count", 3201) @coroutine_test - async def test_stats_calculations(self): + async def test_stats_calculations(self) -> None: logstats = LogStats.from_crawler(self.crawler) with pytest.raises(AttributeError): @@ -56,7 +59,7 @@ class TestLogStats: assert self.stats.get_value("responses_per_minute") == 172.9 assert self.stats.get_value("items_per_minute") == 116.4 - def test_stats_calculations_no_time(self): + def test_stats_calculations_no_time(self) -> None: """The stat values should be None since the start and finish time are not available. """ @@ -65,7 +68,7 @@ class TestLogStats: assert self.stats.get_value("responses_per_minute") is None assert self.stats.get_value("items_per_minute") is None - def test_stats_calculation_no_elapsed_time(self): + def test_stats_calculation_no_elapsed_time(self) -> None: """The stat values should be None since the elapsed time is 0.""" logstats = LogStats.from_crawler(self.crawler) self.stats.set_value("start_time", datetime.fromtimestamp(1655100172)) From cec86f216e02072b2e9ed5b032810e061143277c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 27 Jul 2026 15:19:12 +0500 Subject: [PATCH 008/111] Remove remaining cross-imports in test modules. (#7782) --- pyproject.toml | 3 + tests/mockserver/http_base.py | 8 +- tests/test_command_check.py | 2 +- tests/test_command_crawl.py | 2 +- tests/test_command_genspider.py | 2 +- tests/test_command_parse.py | 2 +- tests/test_commands.py | 2 +- tests/test_downloader_handler_httpx.py | 2 +- .../test_downloader_handler_twisted_http11.py | 2 +- .../test_downloader_handler_twisted_http2.py | 2 +- tests/test_downloadermiddleware_redirect.py | 4 +- ...test_downloadermiddleware_redirect_base.py | 984 ----------------- ...wnloadermiddleware_redirect_metarefresh.py | 4 +- tests/test_engine.py | 337 +----- tests/test_engine_stop_download_bytes.py | 6 +- tests/test_engine_stop_download_headers.py | 6 +- tests/test_feedexport.py | 141 +-- tests/test_feedexport_batch.py | 2 +- tests/test_feedexport_postprocess.py | 2 +- tests/test_http_request.py | 485 +-------- tests/test_http_request_form.py | 4 +- tests/test_http_request_json.py | 4 +- tests/test_http_response.py | 411 +------- tests/test_http_response_text.py | 4 +- tests/test_pipeline_files.py | 4 +- tests/test_pipeline_media.py | 14 +- tests/test_pqueues.py | 2 +- tests/test_scheduler.py | 30 +- tests/test_spider.py | 123 +-- tests/test_spider_crawl.py | 4 +- tests/test_spider_sitemap.py | 4 +- tests/test_spidermiddleware_process_start.py | 3 +- tests/utils/bases/__init__.py | 0 .../{base_commands.py => bases/commands.py} | 0 .../bases/download_handlers_http.py} | 2 - tests/utils/bases/engine.py | 160 +++ tests/utils/bases/feedexport.py | 151 +++ tests/utils/bases/http_request.py | 490 +++++++++ tests/utils/bases/http_response.py | 417 ++++++++ tests/utils/bases/redirect.py | 988 ++++++++++++++++++ tests/utils/bases/spider.py | 131 +++ tests/utils/downloader.py | 35 + tests/utils/engine.py | 184 ++++ tests/utils/media_pipelines.py | 15 + 44 files changed, 2642 insertions(+), 2536 deletions(-) delete mode 100644 tests/test_downloadermiddleware_redirect_base.py create mode 100644 tests/utils/bases/__init__.py rename tests/utils/{base_commands.py => bases/commands.py} (100%) rename tests/{test_downloader_handlers_http_base.py => utils/bases/download_handlers_http.py} (99%) create mode 100644 tests/utils/bases/engine.py create mode 100644 tests/utils/bases/feedexport.py create mode 100644 tests/utils/bases/http_request.py create mode 100644 tests/utils/bases/http_response.py create mode 100644 tests/utils/bases/redirect.py create mode 100644 tests/utils/bases/spider.py create mode 100644 tests/utils/downloader.py create mode 100644 tests/utils/engine.py create mode 100644 tests/utils/media_pipelines.py diff --git a/pyproject.toml b/pyproject.toml index 5ed07a9ce..ee7f4ecdf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -185,6 +185,9 @@ module = [ "tests.test_utils_misc.test_return_with_argument_inside_generator", "tests.test_utils_python", "tests.test_utils_request", + "tests.utils.bases.http_request", + "tests.utils.bases.http_response", + "tests.utils.bases.spider", ] check_untyped_defs = false diff --git a/tests/mockserver/http_base.py b/tests/mockserver/http_base.py index 7b38409ff..343c79781 100644 --- a/tests/mockserver/http_base.py +++ b/tests/mockserver/http_base.py @@ -20,6 +20,9 @@ if TYPE_CHECKING: from twisted.web import resource + # typing.Self requires Python 3.11 + from typing_extensions import Self + class BaseMockServer(ABC): listen_http: bool = True @@ -39,13 +42,14 @@ class BaseMockServer(ABC): self.http_port: int | None = None self.https_port: int | None = None - def __enter__(self): + def __enter__(self) -> Self: self.proc = Popen( [sys.executable, "-u", "-m", self.module_name, *self.get_additional_args()], stdout=PIPE, env=get_script_run_env(), text=True, ) + assert self.proc.stdout is not None if self.listen_http: http_address = self.proc.stdout.readline().strip() http_parsed = urlparse(http_address) @@ -56,7 +60,7 @@ class BaseMockServer(ABC): self.https_port = https_parsed.port return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type, exc_value, traceback) -> None: if self.proc: self.proc.kill() self.proc.communicate() diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 794c6d400..240f44584 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -7,7 +7,7 @@ from unittest import TestCase from unittest.mock import MagicMock, Mock, PropertyMock, call, patch from scrapy.commands.check import Command, TextTestResult -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc if TYPE_CHECKING: diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py index 6293e973e..70c26e6d0 100644 --- a/tests/test_command_crawl.py +++ b/tests/test_command_crawl.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc if TYPE_CHECKING: diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py index f465d0b30..8bb6a2332 100644 --- a/tests/test_command_genspider.py +++ b/tests/test_command_genspider.py @@ -6,7 +6,7 @@ from pathlib import Path import pytest -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import call, proc, write_recording_editor diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index a3581d764..9b7131c7a 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -8,7 +8,7 @@ import pytest from scrapy.commands import parse from scrapy.settings import Settings -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import call, proc if TYPE_CHECKING: diff --git a/tests/test_commands.py b/tests/test_commands.py index 9e7d4d5a1..51f98db1b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -15,7 +15,7 @@ from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.reactor import _asyncio_reactor_path -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import call, proc, write_recording_editor if TYPE_CHECKING: diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index fd50248a7..5ceb93382 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -15,7 +15,7 @@ from scrapy.core.downloader.handlers._httpx import ( HttpxDownloadHandler, ) from scrapy.exceptions import DownloadFailedError -from tests.test_downloader_handlers_http_base import ( +from tests.utils.bases.download_handlers_http import ( TestHttpBase, TestHttpProxyBase, TestHttpsBase, diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index 79750a136..32dfd7540 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -11,7 +11,7 @@ from scrapy import Spider from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from tests.test_downloader_handlers_http_base import ( +from tests.utils.bases.download_handlers_http import ( TestHttpBase, TestHttpProxyBase, TestHttpsBase, diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index bea97642e..daa89df58 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -13,7 +13,7 @@ from scrapy import Spider from scrapy.crawler import Crawler from scrapy.exceptions import DownloadFailedError, NotConfigured from scrapy.http import Request -from tests.test_downloader_handlers_http_base import ( +from tests.utils.bases.download_handlers_http import ( TestHttpProxyBase, TestHttpsBase, TestHttpsCustomCiphersBase, diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 28cf8ba34..ef2774a93 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -16,11 +16,11 @@ from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests.test_downloadermiddleware_redirect_base import Base +from tests.utils.bases.redirect import TestRedirectBase from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMS -class TestRedirectMiddleware(Base.Test): +class TestRedirectMiddleware(TestRedirectBase): mwcls = RedirectMiddleware reason = 302 diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.py deleted file mode 100644 index bd8bc796c..000000000 --- a/tests/test_downloadermiddleware_redirect_base.py +++ /dev/null @@ -1,984 +0,0 @@ -from __future__ import annotations - -import pytest - -from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware -from scrapy.exceptions import IgnoreRequest -from scrapy.http import Request, Response -from scrapy.utils.misc import set_environ -from scrapy.utils.test import get_crawler - - -class Base: - class Test: - def test_priority_adjust(self): - req = Request("http://a.example") - rsp = self.get_response(req, "http://a.example/redirected") - req2 = self.mw.process_response(req, rsp) - assert req2.priority > req.priority - - def test_dont_redirect(self): - url = "http://www.example.com/301" - url2 = "http://www.example.com/redirected" - req = Request(url, meta={"dont_redirect": True}) - rsp = self.get_response(req, url2) - - r = self.mw.process_response(req, rsp) - assert isinstance(r, Response) - assert r is rsp - - # Test that it redirects when dont_redirect is False - req = Request(url, meta={"dont_redirect": False}) - rsp = self.get_response(req, url2) - - r = self.mw.process_response(req, rsp) - assert isinstance(r, Request) - - def test_post(self): - url = "http://www.example.com/302" - url2 = "http://www.example.com/redirected2" - req = Request( - url, - method="POST", - body="test", - headers={"Content-Type": "text/plain", "Content-length": "4"}, - ) - rsp = self.get_response(req, url2) - - req2 = self.mw.process_response(req, rsp) - assert isinstance(req2, Request) - assert req2.url == url2 - assert req2.method == "GET" - assert "Content-Type" not in req2.headers, ( - "Content-Type header must not be present in redirected request" - ) - assert "Content-Length" not in req2.headers, ( - "Content-Length header must not be present in redirected request" - ) - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - - def test_max_redirect_times(self): - self.mw.max_redirect_times = 1 - req = Request("http://a.example/302") - rsp = self.get_response(req, "/redirected") - - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert "redirect_times" in req.meta - assert req.meta["redirect_times"] == 1 - with pytest.raises(IgnoreRequest): - self.mw.process_response(req, rsp) - - def test_ttl(self): - self.mw.max_redirect_times = 100 - req = Request("http://a.example/302", meta={"redirect_ttl": 1}) - rsp = self.get_response(req, "/a") - - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - with pytest.raises(IgnoreRequest): - self.mw.process_response(req, rsp) - - def test_redirect_urls(self): - req1 = Request("http://a.example/first") - rsp1 = self.get_response(req1, "/redirected") - req2 = self.mw.process_response(req1, rsp1) - rsp2 = self.get_response(req2, "/redirected2") - req3 = self.mw.process_response(req2, rsp2) - - assert req2.url == "http://a.example/redirected" - assert req2.meta["redirect_urls"] == ["http://a.example/first"] - assert req3.url == "http://a.example/redirected2" - assert req3.meta["redirect_urls"] == [ - "http://a.example/first", - "http://a.example/redirected", - ] - - def test_redirect_reasons(self): - req1 = Request("http://a.example/first") - rsp1 = self.get_response(req1, "/redirected1") - req2 = self.mw.process_response(req1, rsp1) - rsp2 = self.get_response(req2, "/redirected2") - req3 = self.mw.process_response(req2, rsp2) - assert req2.meta["redirect_reasons"] == [self.reason] - assert req3.meta["redirect_reasons"] == [self.reason, self.reason] - - def test_cross_origin_header_dropping(self): - safe_headers = {"A": "B"} - cookie_header = {"Cookie": "a=b"} - authorization_header = {"Authorization": "Bearer 123456"} - - original_request = Request( - "https://example.com", - headers={**safe_headers, **cookie_header, **authorization_header}, - ) - - # Redirects to the same origin (same scheme, same domain, same port) - # keep all headers. - internal_response = self.get_response( - original_request, "https://example.com/a" - ) - internal_redirect_request = self.mw.process_response( - original_request, internal_response - ) - assert isinstance(internal_redirect_request, Request) - assert original_request.headers == internal_redirect_request.headers - - # Redirects to the same origin (same scheme, same domain, same port) - # keep all headers also when the scheme is http. - http_request = Request( - "http://example.com", - headers={**safe_headers, **cookie_header, **authorization_header}, - ) - http_response = self.get_response(http_request, "http://example.com/a") - http_redirect_request = self.mw.process_response( - http_request, http_response - ) - assert isinstance(http_redirect_request, Request) - assert http_request.headers == http_redirect_request.headers - - # For default ports, whether the port is explicit or implicit does not - # affect the outcome, it is still the same origin. - to_explicit_port_response = self.get_response( - original_request, "https://example.com:443/a" - ) - to_explicit_port_redirect_request = self.mw.process_response( - original_request, to_explicit_port_response - ) - assert isinstance(to_explicit_port_redirect_request, Request) - assert original_request.headers == to_explicit_port_redirect_request.headers - - # For default ports, whether the port is explicit or implicit does not - # affect the outcome, it is still the same origin. - to_implicit_port_response = self.get_response( - original_request, "https://example.com/a" - ) - to_implicit_port_redirect_request = self.mw.process_response( - original_request, to_implicit_port_response - ) - assert isinstance(to_implicit_port_redirect_request, Request) - assert original_request.headers == to_implicit_port_redirect_request.headers - - # A port change drops the Authorization header because the origin - # changes, but keeps the Cookie header because the domain remains the - # same. - different_port_response = self.get_response( - original_request, "https://example.com:8080/a" - ) - different_port_redirect_request = self.mw.process_response( - original_request, different_port_response - ) - assert isinstance(different_port_redirect_request, Request) - assert { - **safe_headers, - **cookie_header, - } == different_port_redirect_request.headers.to_unicode_dict() - - # A domain change drops both the Authorization and the Cookie header. - external_response = self.get_response( - original_request, "https://example.org/a" - ) - external_redirect_request = self.mw.process_response( - original_request, external_response - ) - assert isinstance(external_redirect_request, Request) - assert safe_headers == external_redirect_request.headers.to_unicode_dict() - - # A scheme upgrade (http → https) drops the Authorization header - # because the origin changes, but keeps the Cookie header because the - # domain remains the same. - upgrade_response = self.get_response(http_request, "https://example.com/a") - upgrade_redirect_request = self.mw.process_response( - http_request, upgrade_response - ) - assert isinstance(upgrade_redirect_request, Request) - assert { - **safe_headers, - **cookie_header, - } == upgrade_redirect_request.headers.to_unicode_dict() - - # A scheme downgrade (https → http) drops the Authorization header - # because the origin changes, and the Cookie header because its value - # cannot indicate whether the cookies were secure (HTTPS-only) or not. - # - # Note: If the Cookie header is set by the cookie management - # middleware, as recommended in the docs, the dropping of Cookie on - # scheme downgrade is not an issue, because the cookie management - # middleware will add again the Cookie header to the new request if - # appropriate. - downgrade_response = self.get_response( - original_request, "http://example.com/a" - ) - downgrade_redirect_request = self.mw.process_response( - original_request, downgrade_response - ) - assert isinstance(downgrade_redirect_request, Request) - assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() - - def test_meta_proxy_http_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_http_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_http_to_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_to_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_http_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_http_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_https_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_https_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_proxied_http_to_proxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request2.meta["_auth_proxy"] == "https://b.example" - assert request2.meta["proxy"] == "https://b.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_proxied_http_to_unproxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_unproxied_http_to_proxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request2.meta["_auth_proxy"] == "https://b.example" - assert request2.meta["proxy"] == "https://b.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_unproxied_http_to_unproxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_proxied_https_to_proxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request1.meta["_auth_proxy"] == "https://b.example" - assert request1.meta["proxy"] == "https://b.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request3.meta["_auth_proxy"] == "https://b.example" - assert request3.meta["proxy"] == "https://b.example" - - def test_system_proxy_proxied_https_to_unproxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request1.meta["_auth_proxy"] == "https://b.example" - assert request1.meta["proxy"] == "https://b.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request3.meta["_auth_proxy"] == "https://b.example" - assert request3.meta["proxy"] == "https://b.example" - - def test_system_proxy_unproxied_https_to_proxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_unproxied_https_to_unproxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py index b5d39080c..aeae759a0 100644 --- a/tests/test_downloadermiddleware_redirect_metarefresh.py +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -12,7 +12,7 @@ from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler -from tests.test_downloadermiddleware_redirect_base import Base +from tests.utils.bases.redirect import TestRedirectBase from tests.utils.redirect import ( HTTP_SCHEMES, NON_HTTP_SCHEMES, @@ -26,7 +26,7 @@ def meta_refresh_body(url, interval=5): return html.encode("utf-8") -class TestMetaRefreshMiddleware(Base.Test): +class TestMetaRefreshMiddleware(TestRedirectBase): mwcls = MetaRefreshMiddleware reason = "meta refresh" diff --git a/tests/test_engine.py b/tests/test_engine.py index 3ab8f6d1b..87f0265a0 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,115 +1,45 @@ from __future__ import annotations import asyncio -import re import subprocess import sys -from collections import defaultdict -from dataclasses import dataclass from typing import TYPE_CHECKING, Any from unittest.mock import Mock -from urllib.parse import urlparse -import attr import pytest -from itemadapter import ItemAdapter -from pydispatch import dispatcher from testfixtures import LogCapture -from twisted.internet import defer from scrapy import signals from scrapy.core.engine import ExecutionEngine, _Slot from scrapy.core.scheduler import BaseScheduler from scrapy.exceptions import CloseSpider, IgnoreRequest -from scrapy.http import Headers, Request, Response -from scrapy.item import Field, Item -from scrapy.linkextractors import LinkExtractor +from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.utils.defer import ( - _schedule_coro, - deferred_from_coro, - maybe_deferred_to_future, -) -from scrapy.utils.signal import disconnect_all +from scrapy.utils.defer import _schedule_coro, deferred_from_coro from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests import get_testdata +from tests.utils.bases.engine import TestEngineBase from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.engine import ( + AttrsItemsSpider, + CrawlerRun, + DataClassItemsSpider, + DictItemsSpider, + MySpider, +) if TYPE_CHECKING: from collections.abc import AsyncIterator - from twisted.python.failure import Failure - from tests.mockserver.http import MockServer -class MyItem(Item): - name = Field() - url = Field() - price = Field() - - -@attr.s -class AttrsItem: - name = attr.ib(default="") - url = attr.ib(default="") - price = attr.ib(default=0) - - -@dataclass -class DataClassItem: - name: str = "" - url: str = "" - price: int = 0 - - -class MySpider(Spider): - name = "scrapytest.org" - - itemurl_re = re.compile(r"item\d+.html") - name_re = re.compile(r"

(.*?)

", re.MULTILINE) - price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE) - - item_cls: type = MyItem - - def parse(self, response): - xlink = LinkExtractor() - itemre = re.compile(self.itemurl_re) - for link in xlink.extract_links(response): - if itemre.search(link.url): - yield Request(url=link.url, callback=self.parse_item) - - def parse_item(self, response): - adapter = ItemAdapter(self.item_cls()) - m = self.name_re.search(response.text) - if m: - adapter["name"] = m.group(1) - adapter["url"] = response.url - m = self.price_re.search(response.text) - if m: - adapter["price"] = m.group(1) - return adapter.item - - class DupeFilterSpider(MySpider): async def start(self): for url in self.start_urls: yield Request(url) # no dont_filter=True -class DictItemsSpider(MySpider): - item_cls = dict - - -class AttrsItemsSpider(MySpider): - item_cls = AttrsItem - - -class DataClassItemsSpider(MySpider): - item_cls = DataClassItem - - class ItemZeroDivisionErrorSpider(MySpider): custom_settings = { "ITEM_PIPELINES": { @@ -130,253 +60,6 @@ class ChangeCloseReasonSpider(MySpider): raise CloseSpider(reason="custom_reason") -class CrawlerRun: - """A class to run the crawler and keep track of events occurred""" - - def __init__(self, spider_class: type[Spider]): - self.respplug: list[tuple[Response, Spider]] = [] - self.reqplug: list[tuple[Request, Spider]] = [] - self.reqdropped: list[tuple[Request, Spider]] = [] - self.reqreached: list[tuple[Request, Spider]] = [] - self.itemerror: list[tuple[Any, Response, Spider, Failure]] = [] - self.itemresp: list[tuple[Any, Response]] = [] - self.headers: dict[Request, Headers] = {} - self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list) - self.signals_caught: dict[Any, dict[str, Any]] = {} - self.spider_class = spider_class - - async def run(self, mockserver: MockServer) -> None: - self.mockserver = mockserver - - start_urls = [ - self.geturl("/static/"), - self.geturl("/redirect"), - self.geturl("/redirect"), # duplicate - self.geturl("/numbers"), - ] - - for name, signal in vars(signals).items(): - if not name.startswith("_"): - dispatcher.connect(self.record_signal, signal) - - self.crawler = get_crawler(self.spider_class) - self.crawler.signals.connect(self.item_scraped, signals.item_scraped) - self.crawler.signals.connect(self.item_error, signals.item_error) - self.crawler.signals.connect(self.headers_received, signals.headers_received) - self.crawler.signals.connect(self.bytes_received, signals.bytes_received) - self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) - self.crawler.signals.connect(self.request_dropped, signals.request_dropped) - self.crawler.signals.connect( - self.request_reached, signals.request_reached_downloader - ) - self.crawler.signals.connect( - self.response_downloaded, signals.response_downloaded - ) - self.crawler.crawl(start_urls=start_urls) - - self.deferred: defer.Deferred[None] = defer.Deferred() - dispatcher.connect(self.stop, signals.engine_stopped) - await maybe_deferred_to_future(self.deferred) - - async def stop(self): - for name, signal in vars(signals).items(): - if not name.startswith("_"): - disconnect_all(signal) - self.deferred.callback(None) - await self.crawler.stop_async() - - def geturl(self, path: str) -> str: - return self.mockserver.url(path) - - def getpath(self, url: str) -> str: - u = urlparse(url) - return u.path - - def item_error( - self, item: Any, response: Response, spider: Spider, failure: Failure - ) -> None: - self.itemerror.append((item, response, spider, failure)) - - def item_scraped(self, item: Any, spider: Spider, response: Response) -> None: - self.itemresp.append((item, response)) - - def headers_received( - self, headers: Headers, body_length: int, request: Request, spider: Spider - ) -> None: - self.headers[request] = headers - - def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None: - self.bytes[request].append(data) - - def request_scheduled(self, request: Request, spider: Spider) -> None: - self.reqplug.append((request, spider)) - - def request_reached(self, request: Request, spider: Spider) -> None: - self.reqreached.append((request, spider)) - - def request_dropped(self, request: Request, spider: Spider) -> None: - self.reqdropped.append((request, spider)) - - def response_downloaded(self, response: Response, spider: Spider) -> None: - self.respplug.append((response, spider)) - - def record_signal(self, *args: Any, **kwargs: Any) -> None: - """Record a signal and its parameters""" - signalargs = kwargs.copy() - sig = signalargs.pop("signal") - signalargs.pop("sender", None) - self.signals_caught[sig] = signalargs - - -class TestEngineBase: - @staticmethod - def _assert_visited_urls(run: CrawlerRun) -> None: - must_be_visited = [ - "/static/", - "/redirect", - "/redirected", - "/static/item1.html", - "/static/item2.html", - "/static/item999.html", - ] - urls_visited = {rp[0].url for rp in run.respplug} - urls_expected = {run.geturl(p) for p in must_be_visited} - assert urls_expected <= urls_visited, ( - f"URLs not visited: {list(urls_expected - urls_visited)}" - ) - - @staticmethod - def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None: - assert len(run.reqplug) == count - - paths_expected = [ - "/static/item999.html", - "/static/item2.html", - "/static/item1.html", - ] - - urls_requested = {rq[0].url for rq in run.reqplug} - urls_expected = {run.geturl(p) for p in paths_expected} - assert urls_expected <= urls_requested - scheduled_requests_count = len(run.reqplug) - dropped_requests_count = len(run.reqdropped) - responses_count = len(run.respplug) - assert scheduled_requests_count == dropped_requests_count + responses_count - assert len(run.reqreached) == responses_count - - @staticmethod - def _assert_dropped_requests(run: CrawlerRun) -> None: - assert len(run.reqdropped) == 1 - - @staticmethod - def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None: - # response tests - assert len(run.respplug) == count - assert len(run.reqreached) == count - - for response, _ in run.respplug: - if run.getpath(response.url) == "/static/item999.html": - assert response.status == 404 - if run.getpath(response.url) == "/redirect": - assert response.status == 302 - - @staticmethod - def _assert_items_error(run: CrawlerRun) -> None: - assert len(run.itemerror) == 2 - for item, response, spider, failure in run.itemerror: - assert failure.value.__class__ is ZeroDivisionError - assert spider == run.crawler.spider - - assert item["url"] == response.url - if "item1.html" in item["url"]: - assert item["name"] == "Item 1 name" - assert item["price"] == "100" - if "item2.html" in item["url"]: - assert item["name"] == "Item 2 name" - assert item["price"] == "200" - - @staticmethod - def _assert_scraped_items(run: CrawlerRun) -> None: - assert len(run.itemresp) == 2 - for item_, response in run.itemresp: - item = ItemAdapter(item_) - assert item["url"] == response.url - if "item1.html" in item["url"]: - assert item["name"] == "Item 1 name" - assert item["price"] == "100" - if "item2.html" in item["url"]: - assert item["name"] == "Item 2 name" - assert item["price"] == "200" - - @staticmethod - def _assert_headers_received(run: CrawlerRun) -> None: - for headers in run.headers.values(): - assert b"Server" in headers - assert headers[b"Server"] - assert b"TwistedWeb" in headers[b"Server"] - assert b"Date" in headers - assert b"Content-Type" in headers - - @staticmethod - def _assert_bytes_received(run: CrawlerRun) -> None: - assert len(run.bytes) == 9 - for request, data in run.bytes.items(): - joined_data = b"".join(data) - if run.getpath(request.url) == "/static/": - assert joined_data == get_testdata("test_site", "index.html") - elif run.getpath(request.url) == "/static/item1.html": - assert joined_data == get_testdata("test_site", "item1.html") - elif run.getpath(request.url) == "/static/item2.html": - assert joined_data == get_testdata("test_site", "item2.html") - elif run.getpath(request.url) == "/redirected": - assert joined_data == b"Redirected here" - elif run.getpath(request.url) == "/redirect": - assert ( - joined_data == b"\n\n" - b" \n" - b' \n' - b" \n" - b' \n' - b' click here\n' - b" \n" - b"\n" - ) - elif run.getpath(request.url) == "/static/item999.html": - assert ( - joined_data == b"\n\n" - b" 404 - No Such Resource\n" - b" \n" - b"

No Such Resource

\n" - b"

File not found.

\n" - b" \n" - b"\n" - ) - elif run.getpath(request.url) == "/numbers": - # signal was fired multiple times - assert len(data) > 1 - # bytes were received in order - numbers = [str(x).encode("utf8") for x in range(2**18)] - assert joined_data == b"".join(numbers) - - @staticmethod - def _assert_signals_caught(run: CrawlerRun) -> None: - assert signals.engine_started in run.signals_caught - assert signals.engine_stopped in run.signals_caught - assert signals.spider_opened in run.signals_caught - assert signals.spider_idle in run.signals_caught - assert signals.spider_closed in run.signals_caught - assert signals.headers_received in run.signals_caught - - assert {"spider": run.crawler.spider} == run.signals_caught[ - signals.spider_opened - ] - assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle] - assert { - "spider": run.crawler.spider, - "reason": "finished", - } == run.signals_caught[signals.spider_closed] - - class TestEngine(TestEngineBase): @coroutine_test async def test_crawler(self, mockserver: MockServer) -> None: diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 091485781..27f282af4 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -3,15 +3,15 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy.exceptions import StopDownload -from tests.test_engine import ( +from tests.utils.bases.engine import TestEngineBase +from tests.utils.decorators import coroutine_test +from tests.utils.engine import ( AttrsItemsSpider, CrawlerRun, DataClassItemsSpider, DictItemsSpider, MySpider, - TestEngineBase, ) -from tests.utils.decorators import coroutine_test if TYPE_CHECKING: import pytest diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 8ebc948e2..09fa14326 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -3,15 +3,15 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy.exceptions import StopDownload -from tests.test_engine import ( +from tests.utils.bases.engine import TestEngineBase +from tests.utils.decorators import coroutine_test +from tests.utils.engine import ( AttrsItemsSpider, CrawlerRun, DataClassItemsSpider, DictItemsSpider, MySpider, - TestEngineBase, ) -from tests.utils.decorators import coroutine_test if TYPE_CHECKING: import pytest diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 3f659ee0b..f1c416fbd 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -4,13 +4,9 @@ import csv import json import marshal import pickle -import random -import shutil import tempfile -from abc import ABC, abstractmethod from logging import getLogger from pathlib import Path -from string import ascii_letters, digits from typing import IO, TYPE_CHECKING, Any from unittest import mock @@ -32,8 +28,8 @@ from scrapy.extensions.feedexport import ( ) from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.spiders import ItemSpider +from tests.utils.bases.feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test, inline_callbacks_test from tests.utils.feedexport import MyItem, MyItem2, path_to_url, printf_escape @@ -99,141 +95,6 @@ class LogOnStoreFileStorage: file.close() -class TestFeedExportBase(ABC): - mockserver: MockServer - - def _random_temp_filename(self, inter_dir="") -> Path: - chars = [random.choice(ascii_letters + digits) for _ in range(15)] - filename = "".join(chars) - return Path(self.temp_dir, inter_dir, filename) - - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - def setup_method(self): - self.temp_dir = tempfile.mkdtemp() - - def teardown_method(self): - shutil.rmtree(self.temp_dir, ignore_errors=True) - - async def exported_data( - self, items: Iterable[Any], settings: dict[str, Any] - ) -> dict[str, Any]: - """ - Return exported data which a spider yielding ``items`` would return. - """ - - class TestSpider(scrapy.Spider): - name = "testspider" - - def parse(self, response): - yield from items - - return await self.run_and_export(TestSpider, settings) - - async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]: - """ - Return exported data which a spider yielding no ``items`` would return. - """ - - class TestSpider(scrapy.Spider): - name = "testspider" - - def parse(self, response): - pass - - return await self.run_and_export(TestSpider, settings) - - async def assertExported( - self, - items: Iterable[Any], - header: Iterable[str], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - await self.assertExportedCsv(items, header, rows, settings) - await self.assertExportedJsonLines(items, rows, settings) - await self.assertExportedXml(items, rows, settings) - await self.assertExportedPickle(items, rows, settings) - await self.assertExportedMarshal(items, rows, settings) - await self.assertExportedMultiple(items, rows, settings) - - async def assertExportedCsv( # noqa: B027 - self, - items: Iterable[Any], - header: Iterable[str], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedJsonLines( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedXml( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedMultiple( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedPickle( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedMarshal( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - @abstractmethod - async def run_and_export( - self, spider_cls: type[Spider], settings: dict[str, Any] - ) -> dict[str, Any]: - pass - - def _load_until_eof( - self, data: bytes, load_func: Callable[[IO[bytes]], Any] - ) -> list[Any]: - result: list[Any] = [] - with tempfile.TemporaryFile() as temp: - temp.write(data) - temp.seek(0) - while True: - try: - result.append(load_func(temp)) - except EOFError: - break - return result - - class InstrumentedFeedSlot(FeedSlot): """Instrumented FeedSlot subclass for keeping track of calls to start_exporting and finish_exporting.""" diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 3b50cd492..80ff6229b 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -21,7 +21,7 @@ from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler from tests.spiders import ItemSpider -from tests.test_feedexport import TestFeedExportBase +from tests.utils.bases.feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test, inline_callbacks_test from tests.utils.feedexport import MyItem diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py index 83d976e94..f120ce36f 100644 --- a/tests/test_feedexport_postprocess.py +++ b/tests/test_feedexport_postprocess.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any import pytest from scrapy.utils.test import get_crawler -from tests.test_feedexport import TestFeedExportBase +from tests.utils.bases.feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test from tests.utils.feedexport import path_to_url, printf_escape diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 1941b826f..e58ae8f39 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,493 +1,18 @@ -import warnings import xmlrpc.client -from typing import Any import pytest -from scrapy.http import Headers, Request, XmlRpcRequest -from scrapy.http.request import NO_CALLBACK +from scrapy import Request +from scrapy.http import XmlRpcRequest from scrapy.utils.python import to_bytes +from tests.utils.bases.http_request import TestRequestBase -class TestRequest: +class TestRequest(TestRequestBase): request_class = Request - default_method = "GET" - default_headers: dict[bytes, list[bytes]] = {} - default_meta: dict[str, Any] = {} - def test_init(self): - # Request requires url in the __init__ method - with pytest.raises(TypeError): - self.request_class() - # url argument must be basestring - with pytest.raises(TypeError): - self.request_class(123) - - # priority argument must be an integer - with pytest.raises(TypeError, match="Request priority not an integer"): - self.request_class("http://www.example.com", priority="1") - - r = self.request_class("http://www.example.com") - assert isinstance(r.url, str) - assert r.url == "http://www.example.com" - assert r.method == self.default_method - - assert isinstance(r.headers, Headers) - assert r.headers == self.default_headers - assert r.meta == self.default_meta - - meta = {"lala": "lolo"} - headers = {b"caca": b"coco"} - r = self.request_class( - "http://www.example.com", meta=meta, headers=headers, body="a body" - ) - - assert r.meta is not meta - assert r.meta == meta - assert r.headers is not headers - assert r.headers[b"caca"] == b"coco" - - def test_url_scheme(self): - # This test passes by not raising any (ValueError) exception - self.request_class("http://example.org") - self.request_class("https://example.org") - self.request_class("s3://example.org") - self.request_class("ftp://example.org") - self.request_class("about:config") - self.request_class("data:,Hello%2C%20World!") - - def test_url_no_scheme(self): - msg = "Missing scheme in request url:" - with pytest.raises(ValueError, match=msg): - self.request_class("foo") - with pytest.raises(ValueError, match=msg): - self.request_class("/foo/") - with pytest.raises(ValueError, match=msg): - self.request_class("/foo:bar") - - def test_headers(self): - # Different ways of setting headers attribute - url = "http://www.scrapy.org" - headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"} - r = self.request_class(url=url, headers=headers) - p = self.request_class(url=url, headers=r.headers) - - assert r.headers == p.headers - assert r.headers is not headers - assert p.headers is not r.headers - - # headers must not be unicode - h = Headers({"key1": "val1", "key2": "val2"}) - h["newkey"] = "newval" - for k, v in h.items(): - assert isinstance(k, bytes) - for s in v: - assert isinstance(s, bytes) - - def test_eq(self): - url = "http://www.scrapy.org" - r1 = self.request_class(url=url) - r2 = self.request_class(url=url) - assert r1 != r2 - - set_ = set() - set_.add(r1) - set_.add(r2) - assert len(set_) == 2 - - def test_url(self): - r = self.request_class(url="http://www.scrapy.org/path") - assert r.url == "http://www.scrapy.org/path" - - def test_url_quoting(self): - r = self.request_class(url="http://www.scrapy.org/blank%20space") - assert r.url == "http://www.scrapy.org/blank%20space" - r = self.request_class(url="http://www.scrapy.org/blank space") - assert r.url == "http://www.scrapy.org/blank%20space" - - def test_url_encoding(self): - r = self.request_class(url="http://www.scrapy.org/price/£") - assert r.url == "http://www.scrapy.org/price/%C2%A3" - - def test_url_encoding_other(self): - # encoding affects only query part of URI, not path - # path part should always be UTF-8 encoded before percent-escaping - r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") - assert r.url == "http://www.scrapy.org/price/%C2%A3" - - r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") - assert r.url == "http://www.scrapy.org/price/%C2%A3" - - def test_url_encoding_query(self): - r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") - assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" - - # should be same as above - r2 = self.request_class( - url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8" - ) - assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" - - def test_url_encoding_query_latin1(self): - # encoding is used for encoding query-string before percent-escaping; - # path is still UTF-8 encoded before percent-escaping - r3 = self.request_class( - url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1" - ) - assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3" - - def test_url_encoding_nonutf8_untouched(self): - # percent-escaping sequences that do not match valid UTF-8 sequences - # should be kept untouched (just upper-cased perhaps) - # - # See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2 - # - # "Conversions from URIs to IRIs MUST NOT use any character encoding - # other than UTF-8 in steps 3 and 4, even if it might be possible to - # guess from the context that another character encoding than UTF-8 was - # used in the URI. For example, the URI - # "http://www.example.org/r%E9sum%E9.html" might with some guessing be - # interpreted to contain two e-acute characters encoded as iso-8859-1. - # It must not be converted to an IRI containing these e-acute - # characters. Otherwise, in the future the IRI will be mapped to - # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different - # URI from "http://www.example.org/r%E9sum%E9.html". - r1 = self.request_class(url="http://www.scrapy.org/price/%a3") - assert r1.url == "http://www.scrapy.org/price/%a3" - - r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" - - r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") - assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" - - r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") - assert r4.url == "http://www.example.org/r%E9sum%E9.html" - - def test_url_verbatim(self): - r = self.request_class( - url="http://www.scrapy.org/price/£", - meta={"verbatim_url": True}, - ) - assert r.url == "http://www.scrapy.org/price/£" - - r = self.request_class( - url="http://www.scrapy.org/blank space", - meta={"verbatim_url": True}, - ) - assert r.url == "http://www.scrapy.org/blank space" - - def test_body(self): - r1 = self.request_class(url="http://www.example.com/") - assert r1.body == b"" - - r2 = self.request_class(url="http://www.example.com/", body=b"") - assert isinstance(r2.body, bytes) - assert r2.encoding == "utf-8" # default encoding - - r3 = self.request_class( - url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8" - ) - assert isinstance(r3.body, bytes) - assert r3.body == b"Price: \xc2\xa3100" - - r4 = self.request_class( - url="http://www.example.com/", body="Price: \xa3100", encoding="latin1" - ) - assert isinstance(r4.body, bytes) - assert r4.body == b"Price: \xa3100" - - def test_copy(self): - """Test Request copy""" - - def somecallback(): - pass - - r1 = self.request_class( - "http://www.example.com", - flags=["f1", "f2"], - callback=somecallback, - errback=somecallback, - ) - r1.meta["foo"] = "bar" - r1.cb_kwargs["key"] = "value" - r2 = r1.copy() - - # make sure callbaclks are copied - assert r1.callback is somecallback - assert r1.errback is somecallback - assert r2.callback is r1.callback - assert r2.errback is r1.errback - - # make sure flags list is shallow copied - assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - assert r1.flags == r2.flags - - # make sure cb_kwargs dict is shallow copied - assert r1.cb_kwargs is not r2.cb_kwargs, ( - "cb_kwargs must be a shallow copy, not identical" - ) - assert r1.cb_kwargs == r2.cb_kwargs - - # make sure meta dict is shallow copied - assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" - assert r1.meta == r2.meta - - # make sure headers attribute is shallow copied - assert r1.headers is not r2.headers, ( - "headers must be a shallow copy, not identical" - ) - assert r1.headers == r2.headers - assert r1.encoding == r2.encoding - assert r1.dont_filter == r2.dont_filter - - # Request.body can be identical since it's an immutable object (str) - - def test_copy_inherited_classes(self): - """Test Request children copies preserve their class""" - - class CustomRequest(self.request_class): - pass - - r1 = CustomRequest("http://www.example.com") - r2 = r1.copy() - - assert isinstance(r2, CustomRequest) - - def test_replace(self): - """Test Request.replace() method""" - r1 = self.request_class("http://www.example.com", method="GET") - hdrs = Headers(r1.headers) - hdrs[b"key"] = b"value" - r2 = r1.replace(method="POST", body="New body", headers=hdrs) - assert r1.url == r2.url - assert (r1.method, r2.method) == ("GET", "POST") - assert (r1.body, r2.body) == (b"", b"New body") - assert (r1.headers, r2.headers) == (self.default_headers, hdrs) - - # Empty attributes (which may fail if not compared properly) - r3 = self.request_class( - "http://www.example.com", meta={"a": 1}, dont_filter=True - ) - r4 = r3.replace( - url="http://www.example.com/2", body=b"", meta={}, dont_filter=False - ) - assert r4.url == "http://www.example.com/2" - assert r4.body == b"" - assert r4.meta == {} - assert r4.dont_filter is False - - # the cls argument allows changing the resulting class - custom_request_cls = type("CustomRequest", (self.request_class,), {}) - r5 = r1.replace(cls=custom_request_cls) - assert isinstance(r5, custom_request_cls) - assert r5.url == r1.url - - def test_method_always_str(self): - r = self.request_class("http://www.example.com", method="POST") - assert isinstance(r.method, str) - - def test_immutable_attributes(self): - r = self.request_class("http://example.com") - with pytest.raises(AttributeError): - r.url = "http://example2.com" - with pytest.raises(AttributeError): - r.body = "xxx" - - def test_callback_and_errback(self): - def a_function(): - pass - - r1 = self.request_class("http://example.com") - assert r1.callback is None - assert r1.errback is None - - r2 = self.request_class("http://example.com", callback=a_function) - assert r2.callback is a_function - assert r2.errback is None - - r3 = self.request_class("http://example.com", errback=a_function) - assert r3.callback is None - assert r3.errback is a_function - - r4 = self.request_class( - url="http://example.com", - callback=a_function, - errback=a_function, - ) - assert r4.callback is a_function - assert r4.errback is a_function - - r5 = self.request_class( - url="http://example.com", - callback=NO_CALLBACK, - errback=NO_CALLBACK, - ) - assert r5.callback is NO_CALLBACK - assert r5.errback is NO_CALLBACK - - def test_callback_and_errback_type(self): - with pytest.raises(TypeError): - self.request_class("http://example.com", callback="a_function") - with pytest.raises(TypeError): - self.request_class("http://example.com", errback="a_function") - with pytest.raises(TypeError): - self.request_class( - url="http://example.com", - callback="a_function", - errback="a_function", - ) - - def test_setters(self): - request = self.request_class("http://example.com") - - request.flags = ["f1"] - assert request.flags == ["f1"] - - request.cookies = {"sid": "1"} - assert request.cookies == {"sid": "1"} - - headers = Headers({b"X-Test": b"1"}) - request.headers = headers - assert request._headers is headers - request.headers = {b"A": b"b"} - assert isinstance(request.headers, Headers) - assert request._headers[b"A"] == b"b" - - def test_setter_mutable_lazy_loading(self): - """Mutable attributes are set internally to None only until they are - read, then they always return the same falsy instance of the - corresponding mutable structure. - - Setting them to None causes the next read to return a different object. - """ - - request = self.request_class("http://example.com") - - assert request._flags is None - assert request.flags == [] - assert request.flags is request.flags - assert request._flags == [] - original_flags = request.flags - request.flags = None - assert request._flags is None - assert request.flags == [] - assert request.flags is not original_flags - - assert request._cookies is None - assert request.cookies == {} - assert request.cookies is request.cookies - assert request._cookies == {} - original_cookies = request.cookies - request.cookies = None - assert request._cookies is None - assert request.cookies == {} - assert request.cookies is not original_cookies - - if self.default_headers: - assert request._headers == self.default_headers - assert request._headers is not self.default_headers - assert request.headers == self.default_headers - else: - assert request._headers is None - assert request.headers == {} - assert request.headers is request.headers - assert isinstance(request.headers, Headers) - assert isinstance(request._headers, Headers) - original_headers = request.headers - request.headers = None - assert request._headers is None - assert request.headers == {} - assert request._headers == {} - assert request.headers is not original_headers - - def test_no_callback(self): - with pytest.raises(RuntimeError): - NO_CALLBACK() - - def test_from_curl(self): - # Note: more curated tests regarding curl conversion are in - # `test_utils_curl.py` - curl_command = ( - "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique" - "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique" - "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H" - " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q" - "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use" - "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM" - "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S" - "afari/537.36' -H 'Content-Type: application /x-www-form-urlencode" - "d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=" - "0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0" - "' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee" - "p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith" - "%40example.org&size=small&topping=cheese&topping=onion&delivery=1" - "2%3A15&comments=' --compressed" - ) - r = self.request_class.from_curl(curl_command) - assert r.method == "POST" - assert r.url == "http://httpbin.org/post" - assert ( - r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40" - b"example.org&size=small&topping=cheese&topping=onion" - b"&delivery=12%3A15&comments=" - ) - assert r.cookies == { - "_gauges_unique_year": "1", - "_gauges_unique": "1", - "_gauges_unique_month": "1", - "_gauges_unique_hour": "1", - "_gauges_unique_day": "1", - } - assert r.headers == { - b"Origin": [b"http://httpbin.org"], - b"Accept-Encoding": [b"gzip, deflate"], - b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], - b"Upgrade-Insecure-Requests": [b"1"], - b"User-Agent": [ - b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." - b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" - b".75 Chrome/62.0.3202.75 Safari/537.36" - ], - b"Content-Type": [b"application /x-www-form-urlencoded"], - b"Accept": [ - b"text/html,application/xhtml+xml,application/xml;q=0." - b"9,image/webp,image/apng,*/*;q=0.8" - ], - b"Cache-Control": [b"max-age=0"], - b"Referer": [b"http://httpbin.org/forms/post"], - b"Connection": [b"keep-alive"], - } - - def test_from_curl_with_kwargs(self): - r = self.request_class.from_curl( - 'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"} - ) - assert r.method == "POST" - assert r.meta == {"key": "value"} - - def test_from_curl_ignore_unknown_options(self): - # By default: it works and ignores the unknown options: --foo and -z - with warnings.catch_warnings(): # avoid warning when executing tests - warnings.filterwarnings( - "ignore", category=UserWarning, message="Unrecognized options:" - ) - r = self.request_class.from_curl( - 'curl -X DELETE "http://example.org" --foo -z', - ) - assert r.method == "DELETE" - - # If `ignore_unknown_options` is set to `False` it raises an error with - # the unknown options: --foo and -z - with pytest.raises(ValueError, match="Unrecognized options:"): - self.request_class.from_curl( - 'curl -X PATCH "http://example.org" --foo -z', - ignore_unknown_options=False, - ) - - -class TestXmlRpcRequest(TestRequest): +class TestXmlRpcRequest(TestRequestBase): request_class = XmlRpcRequest default_method = "POST" default_headers = {b"Content-Type": [b"text/xml"]} diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py index af86b35c0..5e965e8dc 100644 --- a/tests/test_http_request_form.py +++ b/tests/test_http_request_form.py @@ -10,7 +10,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import FormRequest, HtmlResponse from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode -from tests.test_http_request import TestRequest +from tests.utils.bases.http_request import TestRequestBase def _buildresponse(body, **kwargs): @@ -31,7 +31,7 @@ def _qs(req, encoding="utf-8", to_unicode=False): # FormRequest.from_response() is deprecated in favor of form2request, so the # many tests below that exercise it ignore the resulting deprecation warning. @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestFormRequest(TestRequest): +class TestFormRequest(TestRequestBase): request_class = FormRequest def assertQueryEqual(self, first, second, msg=None): diff --git a/tests/test_http_request_json.py b/tests/test_http_request_json.py index 65022afe1..1bfbbbdf5 100644 --- a/tests/test_http_request_json.py +++ b/tests/test_http_request_json.py @@ -8,10 +8,10 @@ import pytest from scrapy.http import JsonRequest from scrapy.utils.python import to_bytes -from tests.test_http_request import TestRequest +from tests.utils.bases.http_request import TestRequestBase -class TestJsonRequest(TestRequest): +class TestJsonRequest(TestRequestBase): request_class = JsonRequest default_method = "GET" default_headers = { diff --git a/tests/test_http_response.py b/tests/test_http_response.py index a8ea4920e..369c2495f 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,413 +1,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING - -import pytest -from w3lib.encoding import resolve_encoding - -from scrapy.exceptions import NotSupported -from scrapy.http import Headers, Request, Response -from scrapy.link import Link -from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS -from tests import get_testdata - -if TYPE_CHECKING: - from collections.abc import Iterable +from scrapy.http import Response +from tests.utils.bases.http_response import TestResponseBase -class TestResponse: +class TestResponse(TestResponseBase): response_class = Response - - def test_init(self): - # Response requires url in the constructor - with pytest.raises(TypeError): - self.response_class() - assert isinstance( - self.response_class("http://example.com/"), self.response_class - ) - with pytest.raises(TypeError): - self.response_class(b"http://example.com") - with pytest.raises(TypeError): - self.response_class(url="http://example.com", body={}) - # body can be str or None - assert isinstance( - self.response_class("http://example.com/", body=b""), - self.response_class, - ) - assert isinstance( - self.response_class("http://example.com/", body=b"body"), - self.response_class, - ) - # test presence of all optional parameters - assert isinstance( - self.response_class( - "http://example.com/", body=b"", headers={}, status=200 - ), - self.response_class, - ) - - r = self.response_class("http://www.example.com") - assert isinstance(r.url, str) - assert r.url == "http://www.example.com" - assert r.status == 200 - - assert isinstance(r.headers, Headers) - assert not r.headers - - headers = {"foo": "bar"} - body = b"a body" - r = self.response_class("http://www.example.com", headers=headers, body=body) - - assert r.headers is not headers - assert r.headers[b"foo"] == b"bar" - - r = self.response_class("http://www.example.com", status=301) - assert r.status == 301 - r = self.response_class("http://www.example.com", status="301") - assert r.status == 301 - with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): - self.response_class("http://example.com", status="lala200") - - def test_copy(self): - """Test Response copy""" - - r1 = self.response_class("http://www.example.com", body=b"Some body") - r1.flags.append("cached") - r2 = r1.copy() - - assert r1.status == r2.status - assert r1.body == r2.body - - # make sure flags list is shallow copied - assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - assert r1.flags == r2.flags - - # make sure headers attribute is shallow copied - assert r1.headers is not r2.headers, ( - "headers must be a shallow copy, not identical" - ) - assert r1.headers == r2.headers - - def test_copy_meta(self): - req = Request("http://www.example.com") - req.meta["foo"] = "bar" - r1 = self.response_class( - "http://www.example.com", body=b"Some body", request=req - ) - assert r1.meta is req.meta - - def test_copy_cb_kwargs(self): - req = Request("http://www.example.com") - req.cb_kwargs["foo"] = "bar" - r1 = self.response_class( - "http://www.example.com", body=b"Some body", request=req - ) - assert r1.cb_kwargs is req.cb_kwargs - - def test_unavailable_meta(self): - r1 = self.response_class("http://www.example.com", body=b"Some body") - with pytest.raises(AttributeError, match=r"Response\.meta not available"): - r1.meta - - def test_unavailable_cb_kwargs(self): - r1 = self.response_class("http://www.example.com", body=b"Some body") - with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"): - r1.cb_kwargs - - def test_copy_inherited_classes(self): - """Test Response children copies preserve their class""" - - class CustomResponse(self.response_class): - pass - - r1 = CustomResponse("http://www.example.com") - r2 = r1.copy() - - assert isinstance(r2, CustomResponse) - - def test_replace(self): - """Test Response.replace() method""" - hdrs = Headers({"key": "value"}) - r1 = self.response_class("http://www.example.com") - r2 = r1.replace(status=301, body=b"New body", headers=hdrs) - assert r1.body == b"" - assert r1.url == r2.url - assert (r1.status, r2.status) == (200, 301) - assert (r1.body, r2.body) == (b"", b"New body") - assert (r1.headers, r2.headers) == ({}, hdrs) - - # Empty attributes (which may fail if not compared properly) - r3 = self.response_class("http://www.example.com", flags=["cached"]) - r4 = r3.replace(body=b"", flags=[]) - assert r4.body == b"" - assert not r4.flags - - def _assert_response_values(self, response, encoding, body): - if isinstance(body, str): - body_unicode = body - body_bytes = body.encode(encoding) - else: - body_unicode = body.decode(encoding) - body_bytes = body - - assert isinstance(response.body, bytes) - assert isinstance(response.text, str) - self._assert_response_encoding(response, encoding) - assert response.body == body_bytes - assert response.text == body_unicode - - def _assert_response_encoding(self, response, encoding): - assert response.encoding == resolve_encoding(encoding) - - def test_immutable_attributes(self): - r = self.response_class("http://example.com") - with pytest.raises(AttributeError): - r.url = "http://example2.com" - with pytest.raises(AttributeError): - r.body = "xxx" - - def test_setter_mutable_lazy_loading(self): - """Mutable attributes are set internally to None only until they are - read, then they always return the same falsy instance of the - corresponding mutable structure. - - Setting them to None causes the next read to return a different object. - """ - - response = self.response_class("http://example.com") - - response.request = Request("http://example.com") - - assert response._flags is None - assert response.flags == [] - assert response.flags is response.flags - assert response._flags == [] - original_flags = response.flags - response.flags = None - assert response._flags is None - assert response.flags == [] - assert response.flags is not original_flags - - assert response._headers is None - assert response.headers == {} - assert response.headers is response.headers - assert isinstance(response.headers, Headers) - assert isinstance(response._headers, Headers) - original_headers = response.headers - response.headers = None - assert response._headers is None - assert response.headers == {} - assert response._headers == {} - assert response.headers is not original_headers - - def test_setters(self): - response = self.response_class("http://example.com") - - response.flags = ["f1"] - assert response.flags == ["f1"] - - headers = Headers({b"X-Test": b"1"}) - response.headers = headers - assert response._headers is headers - response.headers = {b"A": b"b"} - assert isinstance(response.headers, Headers) - assert response._headers[b"A"] == b"b" - - def test_urljoin(self): - """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" - joined = self.response_class("http://www.example.com").urljoin("/test") - absolute = "http://www.example.com/test" - assert joined == absolute - - def test_shortcut_attributes(self): - r = self.response_class("http://example.com", body=b"hello") - if self.response_class == Response: - msg = "Response content isn't text" - with pytest.raises(AttributeError, match=msg): - r.text - with pytest.raises(NotSupported, match=msg): - r.css("body") - with pytest.raises(NotSupported, match=msg): - r.xpath("//body") - with pytest.raises(NotSupported, match=msg): - r.jmespath("body") - else: - r.text - r.css("body") - r.xpath("//body") - - # Response.follow - - def test_follow_url_absolute(self): - self._assert_followed_url("http://foo.example.com", "http://foo.example.com") - - def test_follow_url_relative(self): - self._assert_followed_url("foo", "http://example.com/foo") - - def test_follow_link(self): - self._assert_followed_url( - Link("http://example.com/foo"), "http://example.com/foo" - ) - - def test_follow_None_url(self): - r = self.response_class("http://example.com") - with pytest.raises(ValueError, match="url can't be None"): - r.follow(None) - - def test_follow_None_encoding(self): - r = self.response_class("http://example.com") - with pytest.raises(ValueError, match="encoding can't be None"): - r.follow("foo", encoding=None) - - @pytest.mark.xfail( - not W3LIB_STRIPS_URLS, - reason="https://github.com/scrapy/w3lib/pull/207", - strict=True, - ) - def test_follow_whitespace_url(self): - self._assert_followed_url("foo ", "http://example.com/foo") - - @pytest.mark.xfail( - not W3LIB_STRIPS_URLS, - reason="https://github.com/scrapy/w3lib/pull/207", - strict=True, - ) - def test_follow_whitespace_link(self): - self._assert_followed_url( - Link("http://example.com/foo "), "http://example.com/foo" - ) - - def test_follow_flags(self): - res = self.response_class("http://example.com/") - fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - assert fol.flags == ["cached", "allowed"] - - # Response.follow_all - - def test_follow_all_absolute(self): - url_list = [ - "http://example.org", - "http://www.example.org", - "http://example.com", - "http://www.example.com", - ] - self._assert_followed_all_urls(url_list, url_list) - - def test_follow_all_relative(self): - relative = ["foo", "bar", "foo/bar", "bar/foo"] - absolute = [ - "http://example.com/foo", - "http://example.com/bar", - "http://example.com/foo/bar", - "http://example.com/bar/foo", - ] - self._assert_followed_all_urls(relative, absolute) - - def test_follow_all_links(self): - absolute = [ - "http://example.com/foo", - "http://example.com/bar", - "http://example.com/foo/bar", - "http://example.com/bar/foo", - ] - links = map(Link, absolute) - self._assert_followed_all_urls(links, absolute) - - def test_follow_all_empty(self): - r = self.response_class("http://example.com") - assert not list(r.follow_all([])) - - def test_follow_all_invalid(self): - r = self.response_class("http://example.com") - if self.response_class == Response: - with pytest.raises(TypeError): - list(r.follow_all(urls=None)) - with pytest.raises(TypeError): - list(r.follow_all(urls=12345)) - with pytest.raises(ValueError, match="url can't be None"): - list(r.follow_all(urls=[None])) - else: - with pytest.raises( - ValueError, match="Please supply exactly one of the following arguments" - ): - list(r.follow_all(urls=None)) - with pytest.raises(TypeError): - list(r.follow_all(urls=12345)) - with pytest.raises(ValueError, match="url can't be None"): - list(r.follow_all(urls=[None])) - - @pytest.mark.xfail( - not W3LIB_STRIPS_URLS, - reason="https://github.com/scrapy/w3lib/pull/207", - strict=True, - ) - def test_follow_all_whitespace(self): - relative = ["foo ", "bar ", "foo/bar ", "bar/foo "] - absolute = [ - "http://example.com/foo", - "http://example.com/bar", - "http://example.com/foo/bar", - "http://example.com/bar/foo", - ] - self._assert_followed_all_urls(relative, absolute) - - @pytest.mark.xfail( - not W3LIB_STRIPS_URLS, - reason="https://github.com/scrapy/w3lib/pull/207", - strict=True, - ) - def test_follow_all_whitespace_links(self): - absolute = [ - "http://example.com/foo ", - "http://example.com/bar ", - "http://example.com/foo/bar ", - "http://example.com/bar/foo ", - ] - links = [Link(u) for u in absolute] - expected = [u.strip() for u in absolute] - self._assert_followed_all_urls(links, expected) - - def test_follow_all_flags(self): - re = self.response_class("http://www.example.com/") - urls = [ - "http://www.example.com/", - "http://www.example.com/2", - "http://www.example.com/foo", - ] - fol = re.follow_all(urls, flags=["cached", "allowed"]) - for req in fol: - assert req.flags == ["cached", "allowed"] - - def _assert_followed_url( - self, - follow_obj: str | Link, - target_url: str, - response: Response | None = None, - encoding: str | None = None, - ) -> None: - if response is None: - response = self._links_response() - req = response.follow(follow_obj) - assert req.url == target_url - if encoding is not None: - assert req.encoding == encoding - - def _assert_followed_all_urls( - self, - follow_obj: Iterable[str | Link], - target_urls: Iterable[str], - response: Response | None = None, - ) -> None: - if response is None: - response = self._links_response() - followed = response.follow_all(follow_obj) - for req, target in zip(followed, target_urls, strict=True): - assert req.url == target - - def _links_response(self) -> Response: - body = get_testdata("link_extractor", "linkextractor.html") - return self.response_class("http://example.com/index", body=body) - - def _links_response_no_href(self) -> Response: - body = get_testdata("link_extractor", "linkextractor_no_href.html") - return self.response_class("http://example.com/index", body=body) diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index daf6e66e5..04315ad89 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -8,10 +8,10 @@ import pytest from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.selector import Selector from scrapy.utils.python import to_unicode -from tests.test_http_response import TestResponse +from tests.utils.bases.http_response import TestResponseBase -class TestTextResponse(TestResponse): +class TestTextResponse(TestResponseBase): response_class = TextResponse def test_follow_None_encoding(self): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 0d9aac830..a0ae3635b 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -39,8 +39,8 @@ from scrapy.utils.test import get_crawler from tests.mockserver.ftp import MockFTPServer from tests.utils.decorators import coroutine_test, inline_callbacks_test -from .test_pipeline_media import _mocked_download_func from .utils.cloud import mock_google_cloud_storage +from .utils.media_pipelines import mocked_download_func # required by persist_file() and stat_file(), but as some stores don't use the argument # we can pass this singleton to keep type hints correct @@ -94,7 +94,7 @@ class TestFilesPipeline: settings_dict = {"FILES_STORE": self.tempdir} crawler = get_crawler(DefaultSpider, settings_dict=settings_dict) crawler.spider = crawler._create_spider() - crawler.engine = MagicMock(download_async=_mocked_download_func) + crawler.engine = MagicMock(download_async=mocked_download_func) self.pipeline = FilesPipeline.from_crawler(crawler) self.pipeline.open_spider() diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 6862c34eb..ee7a576db 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -9,7 +9,6 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response -from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException from scrapy.pipelines.media import MediaPipeline from scrapy.utils.defer import _defer_sleep_async @@ -18,16 +17,7 @@ from scrapy.utils.signal import disconnect_all from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.utils.decorators import coroutine_test - - -async def _mocked_download_func(request): - assert request.callback is NO_CALLBACK - response = request.meta.get("response") - if callable(response): - response = await response() - if isinstance(response, Exception): - raise response - return response +from tests.utils.media_pipelines import mocked_download_func class UserDefinedPipeline(MediaPipeline): @@ -54,7 +44,7 @@ class TestBaseMediaPipeline: def setup_method(self): crawler = get_crawler(DefaultSpider, self.settings) crawler.spider = crawler._create_spider() - crawler.engine = MagicMock(download_async=_mocked_download_func) + crawler.engine = MagicMock(download_async=mocked_download_func) self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe.open_spider() self.info = self.pipe.spiderinfo diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 6c6a6584a..85fefd172 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -11,7 +11,7 @@ from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler -from tests.test_scheduler import MockDownloader +from tests.utils.downloader import MockDownloader class TestPriorityQueue: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 4fb594444..d7c21bdbd 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -3,7 +3,7 @@ from __future__ import annotations import warnings from abc import ABC, abstractmethod from contextlib import AbstractAsyncContextManager, asynccontextmanager -from typing import TYPE_CHECKING, Any, NamedTuple, cast +from typing import TYPE_CHECKING from unittest.mock import Mock import pytest @@ -15,43 +15,17 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.defer import ensure_awaitable -from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.downloader import MockDownloader if TYPE_CHECKING: from collections.abc import AsyncGenerator from pathlib import Path -class MockSlot(NamedTuple): - active: list[Any] - - -class MockDownloader: - def __init__(self) -> None: - self.slots: dict[str, MockSlot] = {} - - def get_slot_key(self, request: Request) -> str: - if Downloader.DOWNLOAD_SLOT in request.meta: - return cast("str", request.meta[Downloader.DOWNLOAD_SLOT]) - - return urlparse_cached(request).hostname or "" - - def increment(self, slot_key: str) -> None: - slot = self.slots.setdefault(slot_key, MockSlot(active=[])) - slot.active.append(1) - - def decrement(self, slot_key: str) -> None: - slot = self.slots[slot_key] - slot.active.pop() - - def close(self) -> None: - pass - - class MockCrawler(Crawler): def __init__(self, priority_queue_cls: str, jobdir: Path | None): settings = { diff --git a/tests/test_spider.py b/tests/test_spider.py index 7b2727370..03d17199f 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,133 +1,18 @@ from __future__ import annotations -from typing import Any -from unittest import mock - import pytest -from testfixtures import LogCapture -from scrapy import signals -from scrapy.crawler import Crawler -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse, XmlResponse -from scrapy.settings import Settings from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider -from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata -from tests.utils.decorators import inline_callbacks_test +from tests.utils.bases.spider import TestSpiderBase -class TestSpider: +class TestSpider(TestSpiderBase): spider_class = Spider - def test_base_spider(self): - spider = self.spider_class("example.com") - assert spider.name == "example.com" - assert spider.start_urls == [] - def test_spider_args(self): - """``__init__`` method arguments are assigned to spider attributes""" - spider = self.spider_class("example.com", foo="bar") - assert spider.foo == "bar" - - def test_spider_without_name(self): - """``__init__`` raises when the name is not provided.""" - msg = "must have a name" - with pytest.raises(ValueError, match=msg): - self.spider_class() - with pytest.raises(ValueError, match=msg): - self.spider_class(somearg="foo") - - def test_from_crawler_crawler_and_settings_population(self): - crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, "example.com") - assert hasattr(spider, "crawler") - assert spider.crawler is crawler - assert hasattr(spider, "settings") - assert spider.settings is crawler.settings - - def test_from_crawler_init_call(self): - with mock.patch.object( - self.spider_class, "__init__", return_value=None - ) as mock_init: - self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar") - mock_init.assert_called_once_with("example.com", foo="bar") - - def test_closed_signal_call(self): - class TestSpider(self.spider_class): - closed_called = False - - def closed(self, reason): - self.closed_called = True - - crawler = get_crawler() - spider = TestSpider.from_crawler(crawler, "example.com") - crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider) - crawler.signals.send_catch_log( - signal=signals.spider_closed, spider=spider, reason=None - ) - assert spider.closed_called - - def test_update_settings(self): - spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {"TEST1": "project", "TEST3": "project"} - self.spider_class.custom_settings = spider_settings - settings = Settings(project_settings, priority="project") - - self.spider_class.update_settings(settings) - assert settings.get("TEST1") == "spider" - assert settings.get("TEST2") == "spider" - assert settings.get("TEST3") == "project" - - @inline_callbacks_test - def test_settings_in_from_crawler(self): - spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = { - "TEST1": "project", - "TEST3": "project", - **get_reactor_settings(), - } - - class TestSpider(self.spider_class): - name = "test" - custom_settings = spider_settings - - @classmethod - def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): - spider = super().from_crawler(crawler, *args, **kwargs) - spider.settings.set("TEST1", "spider_instance", priority="spider") - return spider - - crawler = Crawler(TestSpider, project_settings) - assert crawler.settings.get("TEST1") == "spider" - assert crawler.settings.get("TEST2") == "spider" - assert crawler.settings.get("TEST3") == "project" - yield crawler.crawl() - assert crawler.settings.get("TEST1") == "spider_instance" - - def test_logger(self): - spider = self.spider_class("example.com") - with LogCapture() as lc: - spider.logger.info("test log msg") - lc.check(("example.com", "INFO", "test log msg")) - - record = lc.records[0] - assert "spider" in record.__dict__ - assert record.spider is spider - - def test_log(self): - spider = self.spider_class("example.com") - with ( - mock.patch("scrapy.spiders.Spider.logger") as mock_logger, - pytest.warns( - ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated" - ), - ): - spider.log("test log msg", "INFO") - mock_logger.log.assert_called_once_with("INFO", "test log msg") - - -class TestXMLFeedSpider(TestSpider): +class TestXMLFeedSpider(TestSpiderBase): spider_class = XMLFeedSpider def test_register_namespace(self): @@ -176,7 +61,7 @@ class TestXMLFeedSpider(TestSpider): ], iterator -class TestCSVFeedSpider(TestSpider): +class TestCSVFeedSpider(TestSpiderBase): spider_class = CSVFeedSpider def test_parse_rows(self): diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py index 63010e195..f34f9add9 100644 --- a/tests/test_spider_crawl.py +++ b/tests/test_spider_crawl.py @@ -11,10 +11,10 @@ from scrapy.http import HtmlResponse, Request, TextResponse from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule, Spider from scrapy.utils.test import get_crawler -from tests.test_spider import TestSpider +from tests.utils.bases.spider import TestSpiderBase -class TestCrawlSpider(TestSpider): +class TestCrawlSpider(TestSpiderBase): test_body = b"""Page title

Item 12

diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py index 0af99ab6d..c8cfd364e 100644 --- a/tests/test_spider_sitemap.py +++ b/tests/test_spider_sitemap.py @@ -14,11 +14,11 @@ from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlRespon from scrapy.spiders import SitemapSpider from scrapy.utils.test import get_crawler from tests import tests_datadir -from tests.test_spider import TestSpider +from tests.utils.bases.spider import TestSpiderBase from tests.utils.decorators import coroutine_test -class TestSitemapSpider(TestSpider): +class TestSitemapSpider(TestSpiderBase): spider_class = SitemapSpider BODY = b"SITEMAP" diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 67ce3a920..321139da3 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -5,7 +5,6 @@ import pytest from scrapy import Spider, signals from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler -from tests.test_spider_start import SLEEP_SECONDS from .utils import twisted_sleep from .utils.decorators import coroutine_test @@ -14,6 +13,8 @@ ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} ITEM_C = {"id": "c"} +SLEEP_SECONDS = 0.1 + class AsyncioSleepSpiderMiddleware: async def process_start(self, start): diff --git a/tests/utils/bases/__init__.py b/tests/utils/bases/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/utils/base_commands.py b/tests/utils/bases/commands.py similarity index 100% rename from tests/utils/base_commands.py rename to tests/utils/bases/commands.py diff --git a/tests/test_downloader_handlers_http_base.py b/tests/utils/bases/download_handlers_http.py similarity index 99% rename from tests/test_downloader_handlers_http_base.py rename to tests/utils/bases/download_handlers_http.py index 29e1d9757..e44f9bcb8 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/utils/bases/download_handlers_http.py @@ -1,5 +1,3 @@ -"""Base classes for HTTP download handler tests.""" - from __future__ import annotations import gzip diff --git a/tests/utils/bases/engine.py b/tests/utils/bases/engine.py new file mode 100644 index 000000000..1cabbb6ef --- /dev/null +++ b/tests/utils/bases/engine.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from itemadapter import ItemAdapter + +from scrapy import signals +from tests import get_testdata + +if TYPE_CHECKING: + from tests.utils.engine import CrawlerRun + + +class TestEngineBase: + @staticmethod + def _assert_visited_urls(run: CrawlerRun) -> None: + must_be_visited = [ + "/static/", + "/redirect", + "/redirected", + "/static/item1.html", + "/static/item2.html", + "/static/item999.html", + ] + urls_visited = {rp[0].url for rp in run.respplug} + urls_expected = {run.geturl(p) for p in must_be_visited} + assert urls_expected <= urls_visited, ( + f"URLs not visited: {list(urls_expected - urls_visited)}" + ) + + @staticmethod + def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None: + assert len(run.reqplug) == count + + paths_expected = [ + "/static/item999.html", + "/static/item2.html", + "/static/item1.html", + ] + + urls_requested = {rq[0].url for rq in run.reqplug} + urls_expected = {run.geturl(p) for p in paths_expected} + assert urls_expected <= urls_requested + scheduled_requests_count = len(run.reqplug) + dropped_requests_count = len(run.reqdropped) + responses_count = len(run.respplug) + assert scheduled_requests_count == dropped_requests_count + responses_count + assert len(run.reqreached) == responses_count + + @staticmethod + def _assert_dropped_requests(run: CrawlerRun) -> None: + assert len(run.reqdropped) == 1 + + @staticmethod + def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None: + # response tests + assert len(run.respplug) == count + assert len(run.reqreached) == count + + for response, _ in run.respplug: + if run.getpath(response.url) == "/static/item999.html": + assert response.status == 404 + if run.getpath(response.url) == "/redirect": + assert response.status == 302 + + @staticmethod + def _assert_items_error(run: CrawlerRun) -> None: + assert len(run.itemerror) == 2 + for item, response, spider, failure in run.itemerror: + assert failure.value.__class__ is ZeroDivisionError + assert spider == run.crawler.spider + + assert item["url"] == response.url + if "item1.html" in item["url"]: + assert item["name"] == "Item 1 name" + assert item["price"] == "100" + if "item2.html" in item["url"]: + assert item["name"] == "Item 2 name" + assert item["price"] == "200" + + @staticmethod + def _assert_scraped_items(run: CrawlerRun) -> None: + assert len(run.itemresp) == 2 + for item_, response in run.itemresp: + item = ItemAdapter(item_) + assert item["url"] == response.url + if "item1.html" in item["url"]: + assert item["name"] == "Item 1 name" + assert item["price"] == "100" + if "item2.html" in item["url"]: + assert item["name"] == "Item 2 name" + assert item["price"] == "200" + + @staticmethod + def _assert_headers_received(run: CrawlerRun) -> None: + for headers in run.headers.values(): + assert b"Server" in headers + assert headers[b"Server"] + assert b"TwistedWeb" in headers[b"Server"] + assert b"Date" in headers + assert b"Content-Type" in headers + + @staticmethod + def _assert_bytes_received(run: CrawlerRun) -> None: + assert len(run.bytes) == 9 + for request, data in run.bytes.items(): + joined_data = b"".join(data) + if run.getpath(request.url) == "/static/": + assert joined_data == get_testdata("test_site", "index.html") + elif run.getpath(request.url) == "/static/item1.html": + assert joined_data == get_testdata("test_site", "item1.html") + elif run.getpath(request.url) == "/static/item2.html": + assert joined_data == get_testdata("test_site", "item2.html") + elif run.getpath(request.url) == "/redirected": + assert joined_data == b"Redirected here" + elif run.getpath(request.url) == "/redirect": + assert ( + joined_data == b"\n\n" + b" \n" + b' \n' + b" \n" + b' \n' + b' click here\n' + b" \n" + b"\n" + ) + elif run.getpath(request.url) == "/static/item999.html": + assert ( + joined_data == b"\n\n" + b" 404 - No Such Resource\n" + b" \n" + b"

No Such Resource

\n" + b"

File not found.

\n" + b" \n" + b"\n" + ) + elif run.getpath(request.url) == "/numbers": + # signal was fired multiple times + assert len(data) > 1 + # bytes were received in order + numbers = [str(x).encode("utf8") for x in range(2**18)] + assert joined_data == b"".join(numbers) + + @staticmethod + def _assert_signals_caught(run: CrawlerRun) -> None: + assert signals.engine_started in run.signals_caught + assert signals.engine_stopped in run.signals_caught + assert signals.spider_opened in run.signals_caught + assert signals.spider_idle in run.signals_caught + assert signals.spider_closed in run.signals_caught + assert signals.headers_received in run.signals_caught + + assert {"spider": run.crawler.spider} == run.signals_caught[ + signals.spider_opened + ] + assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle] + assert { + "spider": run.crawler.spider, + "reason": "finished", + } == run.signals_caught[signals.spider_closed] diff --git a/tests/utils/bases/feedexport.py b/tests/utils/bases/feedexport.py new file mode 100644 index 000000000..f03ac1fd4 --- /dev/null +++ b/tests/utils/bases/feedexport.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import random +import shutil +import tempfile +from abc import ABC, abstractmethod +from pathlib import Path +from string import ascii_letters, digits +from typing import IO, TYPE_CHECKING, Any + +import scrapy +from scrapy import Spider +from tests.mockserver.http import MockServer + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable + + +class TestFeedExportBase(ABC): + mockserver: MockServer + + def _random_temp_filename(self, inter_dir="") -> Path: + chars = [random.choice(ascii_letters + digits) for _ in range(15)] + filename = "".join(chars) + return Path(self.temp_dir, inter_dir, filename) + + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() # pylint: disable=unnecessary-dunder-call + + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) + + def setup_method(self): + self.temp_dir = tempfile.mkdtemp() + + def teardown_method(self): + shutil.rmtree(self.temp_dir, ignore_errors=True) + + async def exported_data( + self, items: Iterable[Any], settings: dict[str, Any] + ) -> dict[str, Any]: + """ + Return exported data which a spider yielding ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = "testspider" + + def parse(self, response): + yield from items + + return await self.run_and_export(TestSpider, settings) + + async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]: + """ + Return exported data which a spider yielding no ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = "testspider" + + def parse(self, response): + pass + + return await self.run_and_export(TestSpider, settings) + + async def assertExported( + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + await self.assertExportedCsv(items, header, rows, settings) + await self.assertExportedJsonLines(items, rows, settings) + await self.assertExportedXml(items, rows, settings) + await self.assertExportedPickle(items, rows, settings) + await self.assertExportedMarshal(items, rows, settings) + await self.assertExportedMultiple(items, rows, settings) + + async def assertExportedCsv( # noqa: B027 + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedJsonLines( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedXml( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedMultiple( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedPickle( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedMarshal( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + @abstractmethod + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, Any]: + pass + + def _load_until_eof( + self, data: bytes, load_func: Callable[[IO[bytes]], Any] + ) -> list[Any]: + result: list[Any] = [] + with tempfile.TemporaryFile() as temp: + temp.write(data) + temp.seek(0) + while True: + try: + result.append(load_func(temp)) + except EOFError: + break + return result diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.py new file mode 100644 index 000000000..c255b5e4c --- /dev/null +++ b/tests/utils/bases/http_request.py @@ -0,0 +1,490 @@ +import warnings +from abc import ABC, abstractmethod +from typing import Any + +import pytest + +from scrapy.http import Headers, Request +from scrapy.http.request import NO_CALLBACK + + +class TestRequestBase(ABC): + default_method = "GET" + default_headers: dict[bytes, list[bytes]] = {} + default_meta: dict[str, Any] = {} + + @property + @abstractmethod + def request_class(self) -> type[Request]: + raise NotImplementedError + + def test_init(self): + # Request requires url in the __init__ method + with pytest.raises(TypeError): + self.request_class() + + # url argument must be basestring + with pytest.raises(TypeError): + self.request_class(123) + + # priority argument must be an integer + with pytest.raises(TypeError, match="Request priority not an integer"): + self.request_class("http://www.example.com", priority="1") + + r = self.request_class("http://www.example.com") + assert isinstance(r.url, str) + assert r.url == "http://www.example.com" + assert r.method == self.default_method + + assert isinstance(r.headers, Headers) + assert r.headers == self.default_headers + assert r.meta == self.default_meta + + meta = {"lala": "lolo"} + headers = {b"caca": b"coco"} + r = self.request_class( + "http://www.example.com", meta=meta, headers=headers, body="a body" + ) + + assert r.meta is not meta + assert r.meta == meta + assert r.headers is not headers + assert r.headers[b"caca"] == b"coco" + + def test_url_scheme(self): + # This test passes by not raising any (ValueError) exception + self.request_class("http://example.org") + self.request_class("https://example.org") + self.request_class("s3://example.org") + self.request_class("ftp://example.org") + self.request_class("about:config") + self.request_class("data:,Hello%2C%20World!") + + def test_url_no_scheme(self): + msg = "Missing scheme in request url:" + with pytest.raises(ValueError, match=msg): + self.request_class("foo") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo/") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo:bar") + + def test_headers(self): + # Different ways of setting headers attribute + url = "http://www.scrapy.org" + headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"} + r = self.request_class(url=url, headers=headers) + p = self.request_class(url=url, headers=r.headers) + + assert r.headers == p.headers + assert r.headers is not headers + assert p.headers is not r.headers + + # headers must not be unicode + h = Headers({"key1": "val1", "key2": "val2"}) + h["newkey"] = "newval" + for k, v in h.items(): + assert isinstance(k, bytes) + for s in v: + assert isinstance(s, bytes) + + def test_eq(self): + url = "http://www.scrapy.org" + r1 = self.request_class(url=url) + r2 = self.request_class(url=url) + assert r1 != r2 + + set_ = set() + set_.add(r1) + set_.add(r2) + assert len(set_) == 2 + + def test_url(self): + r = self.request_class(url="http://www.scrapy.org/path") + assert r.url == "http://www.scrapy.org/path" + + def test_url_quoting(self): + r = self.request_class(url="http://www.scrapy.org/blank%20space") + assert r.url == "http://www.scrapy.org/blank%20space" + r = self.request_class(url="http://www.scrapy.org/blank space") + assert r.url == "http://www.scrapy.org/blank%20space" + + def test_url_encoding(self): + r = self.request_class(url="http://www.scrapy.org/price/£") + assert r.url == "http://www.scrapy.org/price/%C2%A3" + + def test_url_encoding_other(self): + # encoding affects only query part of URI, not path + # path part should always be UTF-8 encoded before percent-escaping + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") + assert r.url == "http://www.scrapy.org/price/%C2%A3" + + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") + assert r.url == "http://www.scrapy.org/price/%C2%A3" + + def test_url_encoding_query(self): + r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") + assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" + + # should be same as above + r2 = self.request_class( + url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8" + ) + assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" + + def test_url_encoding_query_latin1(self): + # encoding is used for encoding query-string before percent-escaping; + # path is still UTF-8 encoded before percent-escaping + r3 = self.request_class( + url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1" + ) + assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3" + + def test_url_encoding_nonutf8_untouched(self): + # percent-escaping sequences that do not match valid UTF-8 sequences + # should be kept untouched (just upper-cased perhaps) + # + # See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2 + # + # "Conversions from URIs to IRIs MUST NOT use any character encoding + # other than UTF-8 in steps 3 and 4, even if it might be possible to + # guess from the context that another character encoding than UTF-8 was + # used in the URI. For example, the URI + # "http://www.example.org/r%E9sum%E9.html" might with some guessing be + # interpreted to contain two e-acute characters encoded as iso-8859-1. + # It must not be converted to an IRI containing these e-acute + # characters. Otherwise, in the future the IRI will be mapped to + # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different + # URI from "http://www.example.org/r%E9sum%E9.html". + r1 = self.request_class(url="http://www.scrapy.org/price/%a3") + assert r1.url == "http://www.scrapy.org/price/%a3" + + r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" + + r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") + assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" + + r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") + assert r4.url == "http://www.example.org/r%E9sum%E9.html" + + def test_url_verbatim(self): + r = self.request_class( + url="http://www.scrapy.org/price/£", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/price/£" + + r = self.request_class( + url="http://www.scrapy.org/blank space", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/blank space" + + def test_body(self): + r1 = self.request_class(url="http://www.example.com/") + assert r1.body == b"" + + r2 = self.request_class(url="http://www.example.com/", body=b"") + assert isinstance(r2.body, bytes) + assert r2.encoding == "utf-8" # default encoding + + r3 = self.request_class( + url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8" + ) + assert isinstance(r3.body, bytes) + assert r3.body == b"Price: \xc2\xa3100" + + r4 = self.request_class( + url="http://www.example.com/", body="Price: \xa3100", encoding="latin1" + ) + assert isinstance(r4.body, bytes) + assert r4.body == b"Price: \xa3100" + + def test_copy(self): + """Test Request copy""" + + def somecallback(): + pass + + r1 = self.request_class( + "http://www.example.com", + flags=["f1", "f2"], + callback=somecallback, + errback=somecallback, + ) + r1.meta["foo"] = "bar" + r1.cb_kwargs["key"] = "value" + r2 = r1.copy() + + # make sure callbaclks are copied + assert r1.callback is somecallback + assert r1.errback is somecallback + assert r2.callback is r1.callback + assert r2.errback is r1.errback + + # make sure flags list is shallow copied + assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" + assert r1.flags == r2.flags + + # make sure cb_kwargs dict is shallow copied + assert r1.cb_kwargs is not r2.cb_kwargs, ( + "cb_kwargs must be a shallow copy, not identical" + ) + assert r1.cb_kwargs == r2.cb_kwargs + + # make sure meta dict is shallow copied + assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" + assert r1.meta == r2.meta + + # make sure headers attribute is shallow copied + assert r1.headers is not r2.headers, ( + "headers must be a shallow copy, not identical" + ) + assert r1.headers == r2.headers + assert r1.encoding == r2.encoding + assert r1.dont_filter == r2.dont_filter + + # Request.body can be identical since it's an immutable object (str) + + def test_copy_inherited_classes(self): + """Test Request children copies preserve their class""" + + class CustomRequest(self.request_class): + pass + + r1 = CustomRequest("http://www.example.com") + r2 = r1.copy() + + assert isinstance(r2, CustomRequest) + + def test_replace(self): + """Test Request.replace() method""" + r1 = self.request_class("http://www.example.com", method="GET") + hdrs = Headers(r1.headers) + hdrs[b"key"] = b"value" + r2 = r1.replace(method="POST", body="New body", headers=hdrs) + assert r1.url == r2.url + assert (r1.method, r2.method) == ("GET", "POST") + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == (self.default_headers, hdrs) + + # Empty attributes (which may fail if not compared properly) + r3 = self.request_class( + "http://www.example.com", meta={"a": 1}, dont_filter=True + ) + r4 = r3.replace( + url="http://www.example.com/2", body=b"", meta={}, dont_filter=False + ) + assert r4.url == "http://www.example.com/2" + assert r4.body == b"" + assert r4.meta == {} + assert r4.dont_filter is False + + # the cls argument allows changing the resulting class + custom_request_cls = type("CustomRequest", (self.request_class,), {}) + r5 = r1.replace(cls=custom_request_cls) + assert isinstance(r5, custom_request_cls) + assert r5.url == r1.url + + def test_method_always_str(self): + r = self.request_class("http://www.example.com", method="POST") + assert isinstance(r.method, str) + + def test_immutable_attributes(self): + r = self.request_class("http://example.com") + with pytest.raises(AttributeError): + r.url = "http://example2.com" + with pytest.raises(AttributeError): + r.body = "xxx" + + def test_callback_and_errback(self): + def a_function(): + pass + + r1 = self.request_class("http://example.com") + assert r1.callback is None + assert r1.errback is None + + r2 = self.request_class("http://example.com", callback=a_function) + assert r2.callback is a_function + assert r2.errback is None + + r3 = self.request_class("http://example.com", errback=a_function) + assert r3.callback is None + assert r3.errback is a_function + + r4 = self.request_class( + url="http://example.com", + callback=a_function, + errback=a_function, + ) + assert r4.callback is a_function + assert r4.errback is a_function + + r5 = self.request_class( + url="http://example.com", + callback=NO_CALLBACK, + errback=NO_CALLBACK, + ) + assert r5.callback is NO_CALLBACK + assert r5.errback is NO_CALLBACK + + def test_callback_and_errback_type(self): + with pytest.raises(TypeError): + self.request_class("http://example.com", callback="a_function") + with pytest.raises(TypeError): + self.request_class("http://example.com", errback="a_function") + with pytest.raises(TypeError): + self.request_class( + url="http://example.com", + callback="a_function", + errback="a_function", + ) + + def test_setters(self): + request = self.request_class("http://example.com") + + request.flags = ["f1"] + assert request.flags == ["f1"] + + request.cookies = {"sid": "1"} + assert request.cookies == {"sid": "1"} + + headers = Headers({b"X-Test": b"1"}) + request.headers = headers + assert request._headers is headers + request.headers = {b"A": b"b"} + assert isinstance(request.headers, Headers) + assert request._headers[b"A"] == b"b" + + def test_setter_mutable_lazy_loading(self): + """Mutable attributes are set internally to None only until they are + read, then they always return the same falsy instance of the + corresponding mutable structure. + + Setting them to None causes the next read to return a different object. + """ + + request = self.request_class("http://example.com") + + assert request._flags is None + assert request.flags == [] + assert request.flags is request.flags + assert request._flags == [] + original_flags = request.flags + request.flags = None + assert request._flags is None + assert request.flags == [] + assert request.flags is not original_flags + + assert request._cookies is None + assert request.cookies == {} + assert request.cookies is request.cookies + assert request._cookies == {} + original_cookies = request.cookies + request.cookies = None + assert request._cookies is None + assert request.cookies == {} + assert request.cookies is not original_cookies + + if self.default_headers: + assert request._headers == self.default_headers + assert request._headers is not self.default_headers + assert request.headers == self.default_headers + else: + assert request._headers is None + assert request.headers == {} + assert request.headers is request.headers + assert isinstance(request.headers, Headers) + assert isinstance(request._headers, Headers) + original_headers = request.headers + request.headers = None + assert request._headers is None + assert request.headers == {} + assert request._headers == {} + assert request.headers is not original_headers + + def test_no_callback(self): + with pytest.raises(RuntimeError): + NO_CALLBACK() + + def test_from_curl(self): + # Note: more curated tests regarding curl conversion are in + # `test_utils_curl.py` + curl_command = ( + "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique" + "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique" + "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H" + " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q" + "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use" + "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM" + "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S" + "afari/537.36' -H 'Content-Type: application /x-www-form-urlencode" + "d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=" + "0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0" + "' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee" + "p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith" + "%40example.org&size=small&topping=cheese&topping=onion&delivery=1" + "2%3A15&comments=' --compressed" + ) + r = self.request_class.from_curl(curl_command) + assert r.method == "POST" + assert r.url == "http://httpbin.org/post" + assert ( + r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40" + b"example.org&size=small&topping=cheese&topping=onion" + b"&delivery=12%3A15&comments=" + ) + assert r.cookies == { + "_gauges_unique_year": "1", + "_gauges_unique": "1", + "_gauges_unique_month": "1", + "_gauges_unique_hour": "1", + "_gauges_unique_day": "1", + } + assert r.headers == { + b"Origin": [b"http://httpbin.org"], + b"Accept-Encoding": [b"gzip, deflate"], + b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], + b"Upgrade-Insecure-Requests": [b"1"], + b"User-Agent": [ + b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." + b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" + b".75 Chrome/62.0.3202.75 Safari/537.36" + ], + b"Content-Type": [b"application /x-www-form-urlencoded"], + b"Accept": [ + b"text/html,application/xhtml+xml,application/xml;q=0." + b"9,image/webp,image/apng,*/*;q=0.8" + ], + b"Cache-Control": [b"max-age=0"], + b"Referer": [b"http://httpbin.org/forms/post"], + b"Connection": [b"keep-alive"], + } + + def test_from_curl_with_kwargs(self): + r = self.request_class.from_curl( + 'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"} + ) + assert r.method == "POST" + assert r.meta == {"key": "value"} + + def test_from_curl_ignore_unknown_options(self): + # By default: it works and ignores the unknown options: --foo and -z + with warnings.catch_warnings(): # avoid warning when executing tests + warnings.filterwarnings( + "ignore", category=UserWarning, message="Unrecognized options:" + ) + r = self.request_class.from_curl( + 'curl -X DELETE "http://example.org" --foo -z', + ) + assert r.method == "DELETE" + + # If `ignore_unknown_options` is set to `False` it raises an error with + # the unknown options: --foo and -z + with pytest.raises(ValueError, match="Unrecognized options:"): + self.request_class.from_curl( + 'curl -X PATCH "http://example.org" --foo -z', + ignore_unknown_options=False, + ) diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.py new file mode 100644 index 000000000..2fbf6527a --- /dev/null +++ b/tests/utils/bases/http_response.py @@ -0,0 +1,417 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +import pytest +from w3lib.encoding import resolve_encoding + +from scrapy.exceptions import NotSupported +from scrapy.http import Headers, Request, Response +from scrapy.link import Link +from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS +from tests import get_testdata + +if TYPE_CHECKING: + from collections.abc import Iterable + + +class TestResponseBase(ABC): + @property + @abstractmethod + def response_class(self) -> type[Response]: + raise NotImplementedError + + def test_init(self): + # Response requires url in the constructor + with pytest.raises(TypeError): + self.response_class() + assert isinstance( + self.response_class("http://example.com/"), self.response_class + ) + with pytest.raises(TypeError): + self.response_class(b"http://example.com") + with pytest.raises(TypeError): + self.response_class(url="http://example.com", body={}) + # body can be str or None + assert isinstance( + self.response_class("http://example.com/", body=b""), + self.response_class, + ) + assert isinstance( + self.response_class("http://example.com/", body=b"body"), + self.response_class, + ) + # test presence of all optional parameters + assert isinstance( + self.response_class( + "http://example.com/", body=b"", headers={}, status=200 + ), + self.response_class, + ) + + r = self.response_class("http://www.example.com") + assert isinstance(r.url, str) + assert r.url == "http://www.example.com" + assert r.status == 200 + + assert isinstance(r.headers, Headers) + assert not r.headers + + headers = {"foo": "bar"} + body = b"a body" + r = self.response_class("http://www.example.com", headers=headers, body=body) + + assert r.headers is not headers + assert r.headers[b"foo"] == b"bar" + + r = self.response_class("http://www.example.com", status=301) + assert r.status == 301 + r = self.response_class("http://www.example.com", status="301") + assert r.status == 301 + with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): + self.response_class("http://example.com", status="lala200") + + def test_copy(self): + """Test Response copy""" + + r1 = self.response_class("http://www.example.com", body=b"Some body") + r1.flags.append("cached") + r2 = r1.copy() + + assert r1.status == r2.status + assert r1.body == r2.body + + # make sure flags list is shallow copied + assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" + assert r1.flags == r2.flags + + # make sure headers attribute is shallow copied + assert r1.headers is not r2.headers, ( + "headers must be a shallow copy, not identical" + ) + assert r1.headers == r2.headers + + def test_copy_meta(self): + req = Request("http://www.example.com") + req.meta["foo"] = "bar" + r1 = self.response_class( + "http://www.example.com", body=b"Some body", request=req + ) + assert r1.meta is req.meta + + def test_copy_cb_kwargs(self): + req = Request("http://www.example.com") + req.cb_kwargs["foo"] = "bar" + r1 = self.response_class( + "http://www.example.com", body=b"Some body", request=req + ) + assert r1.cb_kwargs is req.cb_kwargs + + def test_unavailable_meta(self): + r1 = self.response_class("http://www.example.com", body=b"Some body") + with pytest.raises(AttributeError, match=r"Response\.meta not available"): + r1.meta # pylint: disable=pointless-statement + + def test_unavailable_cb_kwargs(self): + r1 = self.response_class("http://www.example.com", body=b"Some body") + with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"): + r1.cb_kwargs # pylint: disable=pointless-statement + + def test_copy_inherited_classes(self): + """Test Response children copies preserve their class""" + + class CustomResponse(self.response_class): + pass + + r1 = CustomResponse("http://www.example.com") + r2 = r1.copy() + + assert isinstance(r2, CustomResponse) + + def test_replace(self): + """Test Response.replace() method""" + hdrs = Headers({"key": "value"}) + r1 = self.response_class("http://www.example.com") + r2 = r1.replace(status=301, body=b"New body", headers=hdrs) + assert r1.body == b"" + assert r1.url == r2.url + assert (r1.status, r2.status) == (200, 301) + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == ({}, hdrs) + + # Empty attributes (which may fail if not compared properly) + r3 = self.response_class("http://www.example.com", flags=["cached"]) + r4 = r3.replace(body=b"", flags=[]) + assert r4.body == b"" + assert not r4.flags + + def _assert_response_values(self, response, encoding, body): + if isinstance(body, str): + body_unicode = body + body_bytes = body.encode(encoding) + else: + body_unicode = body.decode(encoding) + body_bytes = body + + assert isinstance(response.body, bytes) + assert isinstance(response.text, str) + self._assert_response_encoding(response, encoding) + assert response.body == body_bytes + assert response.text == body_unicode + + def _assert_response_encoding(self, response, encoding): + assert response.encoding == resolve_encoding(encoding) + + def test_immutable_attributes(self): + r = self.response_class("http://example.com") + with pytest.raises(AttributeError): + r.url = "http://example2.com" + with pytest.raises(AttributeError): + r.body = "xxx" + + def test_setter_mutable_lazy_loading(self): + """Mutable attributes are set internally to None only until they are + read, then they always return the same falsy instance of the + corresponding mutable structure. + + Setting them to None causes the next read to return a different object. + """ + + response = self.response_class("http://example.com") + + response.request = Request("http://example.com") + + assert response._flags is None + assert response.flags == [] + assert response.flags is response.flags + assert response._flags == [] + original_flags = response.flags + response.flags = None + assert response._flags is None + assert response.flags == [] + assert response.flags is not original_flags + + assert response._headers is None + assert response.headers == {} + assert response.headers is response.headers + assert isinstance(response.headers, Headers) + assert isinstance(response._headers, Headers) + original_headers = response.headers + response.headers = None + assert response._headers is None + assert response.headers == {} + assert response._headers == {} + assert response.headers is not original_headers + + def test_setters(self): + response = self.response_class("http://example.com") + + response.flags = ["f1"] + assert response.flags == ["f1"] + + headers = Headers({b"X-Test": b"1"}) + response.headers = headers + assert response._headers is headers + response.headers = {b"A": b"b"} + assert isinstance(response.headers, Headers) + assert response._headers[b"A"] == b"b" + + def test_urljoin(self): + """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" + joined = self.response_class("http://www.example.com").urljoin("/test") + absolute = "http://www.example.com/test" + assert joined == absolute + + def test_shortcut_attributes(self): + r = self.response_class("http://example.com", body=b"hello") + if self.response_class == Response: + msg = "Response content isn't text" + with pytest.raises(AttributeError, match=msg): + r.text # pylint: disable=pointless-statement + with pytest.raises(NotSupported, match=msg): + r.css("body") + with pytest.raises(NotSupported, match=msg): + r.xpath("//body") + with pytest.raises(NotSupported, match=msg): + r.jmespath("body") + else: + r.text # pylint: disable=pointless-statement + r.css("body") + r.xpath("//body") + + # Response.follow + + def test_follow_url_absolute(self): + self._assert_followed_url("http://foo.example.com", "http://foo.example.com") + + def test_follow_url_relative(self): + self._assert_followed_url("foo", "http://example.com/foo") + + def test_follow_link(self): + self._assert_followed_url( + Link("http://example.com/foo"), "http://example.com/foo" + ) + + def test_follow_None_url(self): + r = self.response_class("http://example.com") + with pytest.raises(ValueError, match="url can't be None"): + r.follow(None) + + def test_follow_None_encoding(self): + r = self.response_class("http://example.com") + with pytest.raises(ValueError, match="encoding can't be None"): + r.follow("foo", encoding=None) + + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) + def test_follow_whitespace_url(self): + self._assert_followed_url("foo ", "http://example.com/foo") + + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) + def test_follow_whitespace_link(self): + self._assert_followed_url( + Link("http://example.com/foo "), "http://example.com/foo" + ) + + def test_follow_flags(self): + res = self.response_class("http://example.com/") + fol = res.follow("http://example.com/", flags=["cached", "allowed"]) + assert fol.flags == ["cached", "allowed"] + + # Response.follow_all + + def test_follow_all_absolute(self): + url_list = [ + "http://example.org", + "http://www.example.org", + "http://example.com", + "http://www.example.com", + ] + self._assert_followed_all_urls(url_list, url_list) + + def test_follow_all_relative(self): + relative = ["foo", "bar", "foo/bar", "bar/foo"] + absolute = [ + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", + ] + self._assert_followed_all_urls(relative, absolute) + + def test_follow_all_links(self): + absolute = [ + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", + ] + links = map(Link, absolute) + self._assert_followed_all_urls(links, absolute) + + def test_follow_all_empty(self): + r = self.response_class("http://example.com") + assert not list(r.follow_all([])) + + def test_follow_all_invalid(self): + r = self.response_class("http://example.com") + if self.response_class == Response: + with pytest.raises(TypeError): + list(r.follow_all(urls=None)) + with pytest.raises(TypeError): + list(r.follow_all(urls=12345)) + with pytest.raises(ValueError, match="url can't be None"): + list(r.follow_all(urls=[None])) + else: + with pytest.raises( + ValueError, match="Please supply exactly one of the following arguments" + ): + list(r.follow_all(urls=None)) + with pytest.raises(TypeError): + list(r.follow_all(urls=12345)) + with pytest.raises(ValueError, match="url can't be None"): + list(r.follow_all(urls=[None])) + + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) + def test_follow_all_whitespace(self): + relative = ["foo ", "bar ", "foo/bar ", "bar/foo "] + absolute = [ + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", + ] + self._assert_followed_all_urls(relative, absolute) + + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) + def test_follow_all_whitespace_links(self): + absolute = [ + "http://example.com/foo ", + "http://example.com/bar ", + "http://example.com/foo/bar ", + "http://example.com/bar/foo ", + ] + links = [Link(u) for u in absolute] + expected = [u.strip() for u in absolute] + self._assert_followed_all_urls(links, expected) + + def test_follow_all_flags(self): + re = self.response_class("http://www.example.com/") + urls = [ + "http://www.example.com/", + "http://www.example.com/2", + "http://www.example.com/foo", + ] + fol = re.follow_all(urls, flags=["cached", "allowed"]) + for req in fol: + assert req.flags == ["cached", "allowed"] + + def _assert_followed_url( + self, + follow_obj: str | Link, + target_url: str, + response: Response | None = None, + encoding: str | None = None, + ) -> None: + if response is None: + response = self._links_response() + req = response.follow(follow_obj) + assert req.url == target_url + if encoding is not None: + assert req.encoding == encoding + + def _assert_followed_all_urls( + self, + follow_obj: Iterable[str | Link], + target_urls: Iterable[str], + response: Response | None = None, + ) -> None: + if response is None: + response = self._links_response() + followed = response.follow_all(follow_obj) + for req, target in zip(followed, target_urls, strict=True): + assert req.url == target + + def _links_response(self) -> Response: + body = get_testdata("link_extractor", "linkextractor.html") + return self.response_class("http://example.com/index", body=body) + + def _links_response_no_href(self) -> Response: + body = get_testdata("link_extractor", "linkextractor_no_href.html") + return self.response_class("http://example.com/index", body=body) diff --git a/tests/utils/bases/redirect.py b/tests/utils/bases/redirect.py new file mode 100644 index 000000000..e77a30533 --- /dev/null +++ b/tests/utils/bases/redirect.py @@ -0,0 +1,988 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any + +import pytest + +from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware +from scrapy.exceptions import IgnoreRequest +from scrapy.http import Request, Response +from scrapy.utils.misc import set_environ +from scrapy.utils.test import get_crawler + + +class TestRedirectBase(ABC): + mwcls: type[Any] + mw: Any + reason: int | str + + @abstractmethod + def get_response( + self, request: Request, location: str, status: int = 302 + ) -> Response: + raise NotImplementedError + + def test_priority_adjust(self): + req = Request("http://a.example") + rsp = self.get_response(req, "http://a.example/redirected") + req2 = self.mw.process_response(req, rsp) + assert req2.priority > req.priority + + def test_dont_redirect(self): + url = "http://www.example.com/301" + url2 = "http://www.example.com/redirected" + req = Request(url, meta={"dont_redirect": True}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp) + assert isinstance(r, Response) + assert r is rsp + + # Test that it redirects when dont_redirect is False + req = Request(url, meta={"dont_redirect": False}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp) + assert isinstance(r, Request) + + def test_post(self): + url = "http://www.example.com/302" + url2 = "http://www.example.com/redirected2" + req = Request( + url, + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + rsp = self.get_response(req, url2) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == "GET" + assert "Content-Type" not in req2.headers, ( + "Content-Type header must not be present in redirected request" + ) + assert "Content-Length" not in req2.headers, ( + "Content-Length header must not be present in redirected request" + ) + assert not req2.body, f"Redirected body must be empty, not '{req2.body!r}'" + + def test_max_redirect_times(self): + self.mw.max_redirect_times = 1 + req = Request("http://a.example/302") + rsp = self.get_response(req, "/redirected") + + req = self.mw.process_response(req, rsp) + assert isinstance(req, Request) + assert "redirect_times" in req.meta + assert req.meta["redirect_times"] == 1 + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp) + + def test_ttl(self): + self.mw.max_redirect_times = 100 + req = Request("http://a.example/302", meta={"redirect_ttl": 1}) + rsp = self.get_response(req, "/a") + + req = self.mw.process_response(req, rsp) + assert isinstance(req, Request) + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp) + + def test_redirect_urls(self): + req1 = Request("http://a.example/first") + rsp1 = self.get_response(req1, "/redirected") + req2 = self.mw.process_response(req1, rsp1) + rsp2 = self.get_response(req2, "/redirected2") + req3 = self.mw.process_response(req2, rsp2) + + assert req2.url == "http://a.example/redirected" + assert req2.meta["redirect_urls"] == ["http://a.example/first"] + assert req3.url == "http://a.example/redirected2" + assert req3.meta["redirect_urls"] == [ + "http://a.example/first", + "http://a.example/redirected", + ] + + def test_redirect_reasons(self): + req1 = Request("http://a.example/first") + rsp1 = self.get_response(req1, "/redirected1") + req2 = self.mw.process_response(req1, rsp1) + rsp2 = self.get_response(req2, "/redirected2") + req3 = self.mw.process_response(req2, rsp2) + assert req2.meta["redirect_reasons"] == [self.reason] + assert req3.meta["redirect_reasons"] == [self.reason, self.reason] + + def test_cross_origin_header_dropping(self): + safe_headers = {"A": "B"} + cookie_header = {"Cookie": "a=b"} + authorization_header = {"Authorization": "Bearer 123456"} + + original_request = Request( + "https://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers. + internal_response = self.get_response(original_request, "https://example.com/a") + internal_redirect_request = self.mw.process_response( + original_request, internal_response + ) + assert isinstance(internal_redirect_request, Request) + assert original_request.headers == internal_redirect_request.headers + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers also when the scheme is http. + http_request = Request( + "http://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + http_response = self.get_response(http_request, "http://example.com/a") + http_redirect_request = self.mw.process_response(http_request, http_response) + assert isinstance(http_redirect_request, Request) + assert http_request.headers == http_redirect_request.headers + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_explicit_port_response = self.get_response( + original_request, "https://example.com:443/a" + ) + to_explicit_port_redirect_request = self.mw.process_response( + original_request, to_explicit_port_response + ) + assert isinstance(to_explicit_port_redirect_request, Request) + assert original_request.headers == to_explicit_port_redirect_request.headers + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_implicit_port_response = self.get_response( + original_request, "https://example.com/a" + ) + to_implicit_port_redirect_request = self.mw.process_response( + original_request, to_implicit_port_response + ) + assert isinstance(to_implicit_port_redirect_request, Request) + assert original_request.headers == to_implicit_port_redirect_request.headers + + # A port change drops the Authorization header because the origin + # changes, but keeps the Cookie header because the domain remains the + # same. + different_port_response = self.get_response( + original_request, "https://example.com:8080/a" + ) + different_port_redirect_request = self.mw.process_response( + original_request, different_port_response + ) + assert isinstance(different_port_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == different_port_redirect_request.headers.to_unicode_dict() + + # A domain change drops both the Authorization and the Cookie header. + external_response = self.get_response(original_request, "https://example.org/a") + external_redirect_request = self.mw.process_response( + original_request, external_response + ) + assert isinstance(external_redirect_request, Request) + assert safe_headers == external_redirect_request.headers.to_unicode_dict() + + # A scheme upgrade (http → https) drops the Authorization header + # because the origin changes, but keeps the Cookie header because the + # domain remains the same. + upgrade_response = self.get_response(http_request, "https://example.com/a") + upgrade_redirect_request = self.mw.process_response( + http_request, upgrade_response + ) + assert isinstance(upgrade_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == upgrade_redirect_request.headers.to_unicode_dict() + + # A scheme downgrade (https → http) drops the Authorization header + # because the origin changes, and the Cookie header because its value + # cannot indicate whether the cookies were secure (HTTPS-only) or not. + # + # Note: If the Cookie header is set by the cookie management + # middleware, as recommended in the docs, the dropping of Cookie on + # scheme downgrade is not an issue, because the cookie management + # middleware will add again the Cookie header to the new request if + # appropriate. + downgrade_response = self.get_response(original_request, "http://example.com/a") + downgrade_redirect_request = self.mw.process_response( + original_request, downgrade_response + ) + assert isinstance(downgrade_redirect_request, Request) + assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() + + def test_meta_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_http_to_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_to_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_proxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_proxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_unproxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_unproxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_proxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" + + def test_system_proxy_proxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" + + def test_system_proxy_unproxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_unproxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta diff --git a/tests/utils/bases/spider.py b/tests/utils/bases/spider.py new file mode 100644 index 000000000..f774fde07 --- /dev/null +++ b/tests/utils/bases/spider.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any +from unittest import mock + +import pytest +from testfixtures import LogCapture + +from scrapy import signals +from scrapy.crawler import Crawler +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.settings import Settings +from scrapy.utils.test import get_crawler, get_reactor_settings +from tests.utils.decorators import inline_callbacks_test + +if TYPE_CHECKING: + from scrapy.spiders import Spider + + +class TestSpiderBase(ABC): + @property + @abstractmethod + def spider_class(self) -> type[Spider]: + raise NotImplementedError + + def test_base_spider(self): + spider = self.spider_class("example.com") + assert spider.name == "example.com" + assert spider.start_urls == [] + + def test_spider_args(self): + """``__init__`` method arguments are assigned to spider attributes""" + spider = self.spider_class("example.com", foo="bar") + assert spider.foo == "bar" + + def test_spider_without_name(self): + """``__init__`` raises when the name is not provided.""" + msg = "must have a name" + with pytest.raises(ValueError, match=msg): + self.spider_class() + with pytest.raises(ValueError, match=msg): + self.spider_class(somearg="foo") + + def test_from_crawler_crawler_and_settings_population(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + assert hasattr(spider, "crawler") + assert spider.crawler is crawler + assert hasattr(spider, "settings") + assert spider.settings is crawler.settings + + def test_from_crawler_init_call(self): + with mock.patch.object( + self.spider_class, "__init__", return_value=None + ) as mock_init: + self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar") + mock_init.assert_called_once_with("example.com", foo="bar") + + def test_closed_signal_call(self): + class TestSpider(self.spider_class): + closed_called = False + + def closed(self, reason): + self.closed_called = True + + crawler = get_crawler() + spider = TestSpider.from_crawler(crawler, "example.com") + crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider) + crawler.signals.send_catch_log( + signal=signals.spider_closed, spider=spider, reason=None + ) + assert spider.closed_called + + def test_update_settings(self): + spider_settings = {"TEST1": "spider", "TEST2": "spider"} + project_settings = {"TEST1": "project", "TEST3": "project"} + self.spider_class.custom_settings = spider_settings + settings = Settings(project_settings, priority="project") + + self.spider_class.update_settings(settings) + assert settings.get("TEST1") == "spider" + assert settings.get("TEST2") == "spider" + assert settings.get("TEST3") == "project" + + @inline_callbacks_test + def test_settings_in_from_crawler(self): + spider_settings = {"TEST1": "spider", "TEST2": "spider"} + project_settings = { + "TEST1": "project", + "TEST3": "project", + **get_reactor_settings(), + } + + class TestSpider(self.spider_class): + name = "test" + custom_settings = spider_settings + + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("TEST1", "spider_instance", priority="spider") + return spider + + crawler = Crawler(TestSpider, project_settings) + assert crawler.settings.get("TEST1") == "spider" + assert crawler.settings.get("TEST2") == "spider" + assert crawler.settings.get("TEST3") == "project" + yield crawler.crawl() + assert crawler.settings.get("TEST1") == "spider_instance" + + def test_logger(self): + spider = self.spider_class("example.com") + with LogCapture() as lc: + spider.logger.info("test log msg") + lc.check(("example.com", "INFO", "test log msg")) + + record = lc.records[0] + assert "spider" in record.__dict__ + assert record.spider is spider + + def test_log(self): + spider = self.spider_class("example.com") + with ( + mock.patch("scrapy.spiders.Spider.logger") as mock_logger, + pytest.warns( + ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated" + ), + ): + spider.log("test log msg", "INFO") + mock_logger.log.assert_called_once_with("INFO", "test log msg") diff --git a/tests/utils/downloader.py b/tests/utils/downloader.py new file mode 100644 index 000000000..f64c02332 --- /dev/null +++ b/tests/utils/downloader.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, NamedTuple, cast + +from scrapy.core.downloader import Downloader +from scrapy.utils.httpobj import urlparse_cached + +if TYPE_CHECKING: + from scrapy.http import Request + + +class MockSlot(NamedTuple): + active: list[Any] + + +class MockDownloader: + def __init__(self) -> None: + self.slots: dict[str, MockSlot] = {} + + def get_slot_key(self, request: Request) -> str: + if Downloader.DOWNLOAD_SLOT in request.meta: + return cast("str", request.meta[Downloader.DOWNLOAD_SLOT]) + + return urlparse_cached(request).hostname or "" + + def increment(self, slot_key: str) -> None: + slot = self.slots.setdefault(slot_key, MockSlot(active=[])) + slot.active.append(1) + + def decrement(self, slot_key: str) -> None: + slot = self.slots[slot_key] + slot.active.pop() + + def close(self) -> None: + pass diff --git a/tests/utils/engine.py b/tests/utils/engine.py new file mode 100644 index 000000000..c193d39a9 --- /dev/null +++ b/tests/utils/engine.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import re +from collections import defaultdict +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any +from urllib.parse import urlparse + +import attr +from itemadapter import ItemAdapter +from pydispatch import dispatcher +from twisted.internet import defer + +from scrapy import signals +from scrapy.http import Headers, Request, Response +from scrapy.item import Field, Item +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import Spider +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.signal import disconnect_all +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from tests.mockserver.http import MockServer + + +class MyItem(Item): + name = Field() + url = Field() + price = Field() + + +@attr.s +class AttrsItem: + name = attr.ib(default="") + url = attr.ib(default="") + price = attr.ib(default=0) + + +@dataclass +class DataClassItem: + name: str = "" + url: str = "" + price: int = 0 + + +class MySpider(Spider): + name = "scrapytest.org" + + itemurl_re = re.compile(r"item\d+.html") + name_re = re.compile(r"

(.*?)

", re.MULTILINE) + price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE) + + item_cls: type = MyItem + + def parse(self, response): + xlink = LinkExtractor() + itemre = re.compile(self.itemurl_re) + for link in xlink.extract_links(response): + if itemre.search(link.url): + yield Request(url=link.url, callback=self.parse_item) + + def parse_item(self, response): + adapter = ItemAdapter(self.item_cls()) + m = self.name_re.search(response.text) + if m: + adapter["name"] = m.group(1) + adapter["url"] = response.url + m = self.price_re.search(response.text) + if m: + adapter["price"] = m.group(1) + return adapter.item + + +class DictItemsSpider(MySpider): + item_cls = dict + + +class AttrsItemsSpider(MySpider): + item_cls = AttrsItem + + +class DataClassItemsSpider(MySpider): + item_cls = DataClassItem + + +class CrawlerRun: + """A class to run the crawler and keep track of events occurred""" + + def __init__(self, spider_class: type[Spider]): + self.respplug: list[tuple[Response, Spider]] = [] + self.reqplug: list[tuple[Request, Spider]] = [] + self.reqdropped: list[tuple[Request, Spider]] = [] + self.reqreached: list[tuple[Request, Spider]] = [] + self.itemerror: list[tuple[Any, Response, Spider, Failure]] = [] + self.itemresp: list[tuple[Any, Response]] = [] + self.headers: dict[Request, Headers] = {} + self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list) + self.signals_caught: dict[Any, dict[str, Any]] = {} + self.spider_class = spider_class + + async def run(self, mockserver: MockServer) -> None: + self.mockserver = mockserver + + start_urls = [ + self.geturl("/static/"), + self.geturl("/redirect"), + self.geturl("/redirect"), # duplicate + self.geturl("/numbers"), + ] + + for name, signal in vars(signals).items(): + if not name.startswith("_"): + dispatcher.connect(self.record_signal, signal) + + self.crawler = get_crawler(self.spider_class) + self.crawler.signals.connect(self.item_scraped, signals.item_scraped) + self.crawler.signals.connect(self.item_error, signals.item_error) + self.crawler.signals.connect(self.headers_received, signals.headers_received) + self.crawler.signals.connect(self.bytes_received, signals.bytes_received) + self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) + self.crawler.signals.connect(self.request_dropped, signals.request_dropped) + self.crawler.signals.connect( + self.request_reached, signals.request_reached_downloader + ) + self.crawler.signals.connect( + self.response_downloaded, signals.response_downloaded + ) + self.crawler.crawl(start_urls=start_urls) + + self.deferred: defer.Deferred[None] = defer.Deferred() + dispatcher.connect(self.stop, signals.engine_stopped) + await maybe_deferred_to_future(self.deferred) + + async def stop(self): + for name, signal in vars(signals).items(): + if not name.startswith("_"): + disconnect_all(signal) + self.deferred.callback(None) + await self.crawler.stop_async() + + def geturl(self, path: str) -> str: + return self.mockserver.url(path) + + def getpath(self, url: str) -> str: + u = urlparse(url) + return u.path + + def item_error( + self, item: Any, response: Response, spider: Spider, failure: Failure + ) -> None: + self.itemerror.append((item, response, spider, failure)) + + def item_scraped(self, item: Any, spider: Spider, response: Response) -> None: + self.itemresp.append((item, response)) + + def headers_received( + self, headers: Headers, body_length: int, request: Request, spider: Spider + ) -> None: + self.headers[request] = headers + + def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None: + self.bytes[request].append(data) + + def request_scheduled(self, request: Request, spider: Spider) -> None: + self.reqplug.append((request, spider)) + + def request_reached(self, request: Request, spider: Spider) -> None: + self.reqreached.append((request, spider)) + + def request_dropped(self, request: Request, spider: Spider) -> None: + self.reqdropped.append((request, spider)) + + def response_downloaded(self, response: Response, spider: Spider) -> None: + self.respplug.append((response, spider)) + + def record_signal(self, *args: Any, **kwargs: Any) -> None: + """Record a signal and its parameters""" + signalargs = kwargs.copy() + sig = signalargs.pop("signal") + signalargs.pop("sender", None) + self.signals_caught[sig] = signalargs diff --git a/tests/utils/media_pipelines.py b/tests/utils/media_pipelines.py new file mode 100644 index 000000000..283c95940 --- /dev/null +++ b/tests/utils/media_pipelines.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from typing import Any + +from scrapy.http.request import NO_CALLBACK, Request + + +async def mocked_download_func(request: Request) -> Any: + assert request.callback is NO_CALLBACK + response = request.meta.get("response") + if callable(response): + response = await response() + if isinstance(response, Exception): + raise response + return response From 25e6884e2f24d554e3527cbe20ff001913cc5863 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 27 Jul 2026 17:34:10 +0200 Subject: [PATCH 009/111] Address recursion and media ignore-request reporting issues (#7673) * Address recursion and media ignore-request reporting issues * Improve test coverage * Address feedback --- scrapy/core/engine.py | 21 ++++--- scrapy/downloadermiddlewares/offsite.py | 2 +- scrapy/pipelines/files.py | 36 ++++++----- scrapy/pipelines/media.py | 24 +++++++- tests/test_downloadermiddleware_offsite.py | 14 +++++ tests/test_engine_download.py | 20 ++++++ tests/test_pipeline_files.py | 72 +++++++++++++++++++++- tests/test_pipeline_media.py | 18 +++++- 8 files changed, 176 insertions(+), 31 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 1033e874f..efac5f71c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -470,16 +470,17 @@ class ExecutionEngine: """ if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - try: - response_or_request = await maybe_deferred_to_future( - self._download(request) - ) - finally: - assert self._slot is not None - self._slot.remove_request(request) - if isinstance(response_or_request, Request): - return await self.download_async(response_or_request) - return response_or_request + while True: + try: + response_or_request = await maybe_deferred_to_future( + self._download(request) + ) + finally: + assert self._slot is not None + self._slot.remove_request(request) + if not isinstance(response_or_request, Request): + return response_or_request + request = response_or_request @inlineCallbacks def _download( diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index e03dc040e..db85b62a1 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -61,7 +61,7 @@ class OffsiteMiddleware: ) self.stats.inc_value("offsite/domains") self.stats.inc_value("offsite/filtered") - raise IgnoreRequest + raise IgnoreRequest(f"Filtered offsite request to {domain!r}") def should_follow(self, request: Request, spider: Spider) -> bool: regex = self.host_regex diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 70dc36e35..8e8082332 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -27,7 +27,15 @@ from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline +from scrapy.pipelines.media import ( + FileException as FileException, # noqa: PLC0414 # re-exported for backward compatibility +) +from scrapy.pipelines.media import ( + FileInfo, + FileInfoOrError, + MediaPipeline, + _MediaRequestFiltered, +) from scrapy.utils.asyncio import run_in_thread from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict @@ -75,10 +83,6 @@ def _md5sum(file: IO[bytes]) -> str: return m.hexdigest() -class FileException(Exception): - """General media error exception""" - - class StatInfo(TypedDict, total=False): checksum: str last_modified: float @@ -597,20 +601,20 @@ class FilesPipeline(MediaPipeline): def media_failed( self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo ) -> NoReturn: - if not isinstance(failure.value, IgnoreRequest): - referer = referer_str(request) - logger.warning( - "File (unknown-error): Error downloading %(medianame)s from " - "%(request)s referred in <%(referer)s>: %(exception)s", - { - "medianame": self.MEDIA_NAME, - "request": request, - "referer": referer, - "exception": failure.value, - }, + referer = referer_str(request) + if isinstance(failure.value, IgnoreRequest): + logger.debug( + f"File (filtered): Not downloading {self.MEDIA_NAME} from " + f"{request} referred in <{referer}>: {failure.value}", extra={"spider": info.spider}, ) + raise _MediaRequestFiltered(str(failure.value)) from failure.value + logger.warning( + f"File (unknown-error): Error downloading {self.MEDIA_NAME} from " + f"{request} referred in <{referer}>: {failure.value}", + extra={"spider": info.spider}, + ) raise FileException async def media_downloaded( diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 5b5d2dcb2..764f82a78 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -55,6 +55,25 @@ FileInfoOrError: TypeAlias = ( logger = logging.getLogger(__name__) +class FileException(Exception): + """General media error exception""" + + +class _MediaRequestFiltered(FileException): + """Raised internally by media pipelines when a media request is filtered + out (e.g. as an offsite request) instead of being downloaded. + + It is a subclass of :exc:`FileException` for backward compatibility, but + unlike an actual download error it is logged at the ``DEBUG`` level and + without a traceback, since filtering a request is expected behavior rather + than an error. + """ + + +def _media_request_filtered(failure: Failure) -> bool: + return isinstance(failure.value, _MediaRequestFiltered) + + class MediaPipeline(ABC): LOG_FAILED_RESULTS: bool = True @@ -193,7 +212,8 @@ class MediaPipeline(ABC): result = await self._check_media_to_download(request, info, item=item) except Exception: result = Failure() - logger.exception(result) + if not _media_request_filtered(result): + logger.exception(result) self._cache_result_and_execute_waiters(result, fp, info) return await maybe_deferred_to_future(wad) # it must return wad at last @@ -304,6 +324,8 @@ class MediaPipeline(ABC): for ok, value in results: if not ok: assert isinstance(value, Failure) + if _media_request_filtered(value): + continue logger.error( "%(class)s found errors processing %(item)s", {"class": self.__class__.__name__, "item": item}, diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index c0b8dc4dd..78efb0191 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -1,3 +1,5 @@ +import re + import pytest from scrapy import Request, Spider @@ -231,3 +233,15 @@ def test_repeated_offsite_domain(): mw.process_request(req2) assert crawler.stats.get_value("offsite/domains") == 1 # not incremented again assert crawler.stats.get_value("offsite/filtered") == 2 + + +def test_ignore_request_reason(): + crawler = get_crawler(Spider) + crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(crawler.spider) + request = Request("http://other.org/1") + with pytest.raises( + IgnoreRequest, match=re.escape("Filtered offsite request to 'other.org'") + ): + mw.process_request(request) diff --git a/tests/test_engine_download.py b/tests/test_engine_download.py index f15bfd5e2..962808d96 100644 --- a/tests/test_engine_download.py +++ b/tests/test_engine_download.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys from unittest.mock import Mock, call import pytest @@ -73,6 +74,25 @@ class TestEngineDownloadAsync: [call(original_request), call(redirect_request)] ) + @coroutine_test + async def test_download_async_many_redirects(self, engine): + """A long chain of requests being replaced by new ones is handled + iteratively, without hitting the recursion limit.""" + count = sys.getrecursionlimit() * 2 + requests = [Request(f"http://example.com/{i}") for i in range(count)] + final_response = Response("http://example.com/final", body=b"done") + engine.downloader.fetch.side_effect = [ + *(defer.succeed(request) for request in requests[1:]), + defer.succeed(final_response), + ] + engine.spider = Mock() + engine._slot.add_request = Mock() + engine._slot.remove_request = Mock() + + result = await self._download(engine, requests[0]) + assert result == final_response + assert engine.downloader.fetch.call_count == count + @coroutine_test async def test_download_async_no_spider(self, engine): """Test async download attempt when no spider is available.""" diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index a0ae3635b..4f6fa21a0 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -2,6 +2,7 @@ import base64 import dataclasses import logging import random +import re import time from abc import ABC, abstractmethod from datetime import datetime @@ -19,18 +20,20 @@ import attr import pytest from itemadapter import ItemAdapter from twisted.internet.defer import Deferred +from twisted.python.failure import Failure -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.item import Field, Item from scrapy.pipelines.files import ( + FileException, FilesPipeline, FSFilesStore, FTPFilesStore, GCSFilesStore, S3FilesStore, ) -from scrapy.pipelines.media import MediaPipeline +from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered from scrapy.settings import Settings from scrapy.utils.asyncio import call_later from scrapy.utils.defer import maybe_deferred_to_future @@ -290,6 +293,71 @@ class TestFilesPipeline: request = Request("http://example.com") assert file_path(request, item=item) == "full/path-to-store-file" + def test_media_failed_filtered_request( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """A filtered media request (IgnoreRequest) is reported as a + _MediaRequestFiltered exception and logged at the DEBUG level, instead + of as a download error with a traceback.""" + request = Request("http://example.com/file.pdf") + reason = "Filtered offsite request to 'example.com'" + failure = Failure(IgnoreRequest(reason)) + + with ( + caplog.at_level(logging.DEBUG), + pytest.raises(_MediaRequestFiltered, match=re.escape(reason)), + ): + self.pipeline.media_failed(failure, request, self.pipeline.spiderinfo) + + assert len(caplog.records) == 1 + record = caplog.records[0] + assert record.levelname == "DEBUG" + assert record.exc_info is None + assert reason in record.getMessage() + + def test_media_failed_download_error( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """A genuine download error is reported as a FileException and logged as + a warning.""" + request = Request("http://example.com/file.pdf") + failure = Failure(Exception("boom")) + + with caplog.at_level(logging.WARNING), pytest.raises(FileException): + self.pipeline.media_failed(failure, request, self.pipeline.spiderinfo) + + assert len(caplog.records) == 1 + assert caplog.records[0].levelname == "WARNING" + + @coroutine_test + async def test_process_item_filtered_request( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """A filtered (e.g. offsite) media request is processed as a failed + result without being logged as an error with a traceback.""" + item_url = "http://example.com/file.pdf" + item = _create_item_with_files(item_url) + request = Request( + item_url, + meta={ + "response": IgnoreRequest("Filtered offsite request to 'example.com'") + }, + ) + with ( + caplog.at_level(logging.DEBUG), + mock.patch.object( + FilesPipeline, "get_media_requests", return_value=[request] + ), + ): + result = await self.pipeline.process_item(item) + + assert result["files"] == [] + assert not any(r.levelname in ("WARNING", "ERROR") for r in caplog.records) + assert any( + "Filtered offsite request to 'example.com'" in r.getMessage() + for r in caplog.records + ) + @pytest.mark.parametrize( "bad_type", [ diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index ee7a576db..be471e5fe 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from unittest.mock import MagicMock import pytest @@ -10,7 +11,7 @@ from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.pipelines.files import FileException -from scrapy.pipelines.media import MediaPipeline +from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered from scrapy.utils.defer import _defer_sleep_async from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all @@ -152,6 +153,21 @@ class TestBaseMediaPipeline: assert new_item is item assert len(log.records) == 0 + def test_item_completed_filtered_request_not_logged( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Filtered media requests (e.g. offsite ones) are not logged as errors + by item_completed(), as they are not download errors.""" + item = {"name": "name"} + fail = Failure(_MediaRequestFiltered("Filtered offsite request")) + results = [(True, 1), (False, fail)] + + with caplog.at_level(logging.DEBUG): + new_item = self.pipe.item_completed(results, item, self.info) + + assert new_item is item + assert len(caplog.records) == 0 + @coroutine_test async def test_default_process_item(self): item = {"name": "name"} From f3693aa8ba7be410c5d892eaf5d11580d6a969b2 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 27 Jul 2026 18:23:56 +0200 Subject: [PATCH 010/111] Fix port issue with cached DNS (#7772) * Fix port issue with cached DNS * Keep ports off the cache * Complete coverage --- scrapy/resolver.py | 17 +++++++++-- tests/test_resolver.py | 69 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 270a7fbf5..0e9775fd1 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -2,6 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any +import attr from twisted.internet import defer from twisted.internet.base import ReactorBase, ThreadedResolver from twisted.internet.interfaces import ( @@ -70,6 +71,12 @@ class CachingThreadedResolver(ThreadedResolver): return result +def _address_with_port(address: IAddress, port: int) -> IAddress: + if getattr(address, "port", port) == port: + return address + return attr.evolve(address, port=port) + + @implementer(IHostResolution) class HostResolution: def __init__(self, name: str): @@ -97,7 +104,11 @@ class _CachingResolutionReceiver: def resolutionComplete(self) -> None: self.resolutionReceiver.resolutionComplete() if self.addresses: - dnscache[self.hostName] = self.addresses + # Name resolution does not depend on the port, so cache entries are + # kept port-agnostic and the requested port is set on cache hits. + dnscache[self.hostName] = [ + _address_with_port(address, 0) for address in self.addresses + ] @implementer(IHostnameResolver) @@ -142,7 +153,7 @@ class CachingHostnameResolver: transportSemantics, ) resolutionReceiver.resolutionBegan(HostResolution(hostName)) - for addr in addresses: - resolutionReceiver.addressResolved(addr) + for address in addresses: + resolutionReceiver.addressResolved(_address_with_port(address, portNumber)) resolutionReceiver.resolutionComplete() return resolutionReceiver diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 7cca45ed1..8e69f62a0 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -3,6 +3,7 @@ from __future__ import annotations from unittest.mock import Mock import pytest +from twisted.internet.address import IPv4Address, IPv6Address from scrapy.resolver import CachingHostnameResolver, CachingThreadedResolver, dnscache from scrapy.utils.defer import maybe_deferred_to_future @@ -55,11 +56,77 @@ def test_caching_hostname_resolver_no_addresses_not_cached(): assert "example.com" not in dnscache +def test_caching_hostname_resolver_cached_addresses_have_no_port(): + def fake_resolve(receiver, *_): + receiver.resolutionBegan(Mock()) + receiver.addressResolved(IPv4Address("TCP", "1.2.3.4", 80)) + receiver.addressResolved(IPv6Address("TCP", "::1", 80)) + receiver.resolutionComplete() + return receiver + + reactor = Mock() + reactor.nameResolver.resolveHostName.side_effect = fake_resolve + + receiver = Mock() + resolver = CachingHostnameResolver(reactor, cache_size=10) + resolver.resolveHostName(receiver, "example.com", portNumber=80) + + # The port requested on a cache miss is passed through unchanged, but it is + # not part of what gets cached. + resolved_ports = [ + call.args[0].port for call in receiver.addressResolved.call_args_list + ] + assert resolved_ports == [80, 80] + assert [address.port for address in dnscache["example.com"]] == [0, 0] + + +def test_caching_hostname_resolver_cache_hit_without_port(): + cached_addresses = [ + IPv4Address("TCP", "1.2.3.4", 0), + IPv6Address("TCP", "::1", 0), + ] + dnscache["example.com"] = cached_addresses + + receiver = Mock() + resolver = CachingHostnameResolver(Mock(), cache_size=10) + resolver.resolveHostName(receiver, "example.com") + + # Cached addresses already use the requested port, so they are reused as is. + resolved_addresses = [ + call.args[0] for call in receiver.addressResolved.call_args_list + ] + assert all( + resolved is cached + for resolved, cached in zip(resolved_addresses, cached_addresses, strict=True) + ) + + +def test_caching_hostname_resolver_cache_hit_uses_requested_port(): + dnscache["example.com"] = [ + IPv4Address("TCP", "1.2.3.4", 0), + IPv6Address("TCP", "::1", 0), + ] + + receiver = Mock() + resolver = CachingHostnameResolver(Mock(), cache_size=10) + resolver.resolveHostName(receiver, "example.com", portNumber=443) + + resolved_addresses = [ + call.args[0] for call in receiver.addressResolved.call_args_list + ] + assert resolved_addresses == [ + IPv4Address("TCP", "1.2.3.4", 443), + IPv6Address("TCP", "::1", 443), + ] + # The cached addresses must not be mutated in place. + assert [address.port for address in dnscache["example.com"]] == [0, 0] + + def test_caching_hostname_resolver_dnscache_disabled_rejects_storage(): def fake_resolve(receiver, *_): receiver.resolutionBegan(Mock()) - receiver.addressResolved(Mock()) + receiver.addressResolved(IPv4Address("TCP", "1.2.3.4", 80)) receiver.resolutionComplete() return receiver From a5bc43e34c301232571708615f8b8d78cb56a36d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 Jul 2026 12:51:52 +0500 Subject: [PATCH 011/111] Don't generate test keys concurrently with xdist. (#7792) --- conftest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index b8a9dc19e..27c398792 100644 --- a/conftest.py +++ b/conftest.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from importlib.util import find_spec from pathlib import Path from typing import TYPE_CHECKING @@ -135,5 +136,6 @@ def pytest_runtest_setup(item): pytest.skip("mitmdump is not available") -# Generate localhost certificate files, needed by some tests -generate_keys() +# Generate localhost certificate files, needed by some tests (but only once if xdist is used) +if "PYTEST_XDIST_WORKER" not in os.environ: + generate_keys() From 7e8b58a2b2c0c2cf92a81dd7079580b8c603751c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 Jul 2026 12:52:52 +0500 Subject: [PATCH 012/111] Migrate away from testfixtures. (#7793) --- pyproject.toml | 1 - .../test_downloader_handler_twisted_http2.py | 24 +- tests/test_downloadermiddleware_cookies.py | 126 ++++--- ...st_downloadermiddleware_httpcompression.py | 188 ++++++---- tests/test_downloadermiddleware_retry.py | 350 +++++++++--------- tests/test_dupefilters.py | 156 ++++---- tests/test_engine.py | 16 +- tests/test_feedexport.py | 26 +- tests/test_feedexport_storages.py | 59 ++- tests/test_logformatter.py | 51 ++- tests/test_pipeline_crawl.py | 74 ++-- tests/test_pipeline_media.py | 18 +- tests/test_request_attribute_binding.py | 60 +-- tests/test_request_cb_kwargs.py | 52 +-- tests/test_scheduler_base.py | 34 +- tests/test_spider_sitemap.py | 57 +-- tests/test_spidermiddleware_output_chain.py | 137 ++++--- tests/test_utils_log.py | 9 +- tests/test_utils_signal.py | 44 +-- tests/utils/bases/spider.py | 14 +- tox.ini | 1 - 21 files changed, 775 insertions(+), 722 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ee7f4ecdf..9b1e64121 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,7 +225,6 @@ module = [ "pyftpdlib.*", "pytest_twisted", "robotexclusionrulesparser", - "testfixtures", "zope.interface.*", ] ignore_missing_imports = true diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index daa89df58..449f2d635 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -2,11 +2,11 @@ from __future__ import annotations +import logging import sys from typing import TYPE_CHECKING, Any import pytest -from testfixtures import LogCapture from twisted.web.http import H2_ENABLED from scrapy import Spider @@ -130,24 +130,24 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase): assert response.text == custom_content_length @coroutine_test - async def test_custom_content_length_bad(self, mockserver: MockServer) -> None: + async def test_custom_content_length_bad( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: request = Request(mockserver.url("/contentlength", is_secure=self.is_secure)) actual_content_length = str(len(request.body)) bad_content_length = str(len(request.body) + 1) request.headers["Content-Length"] = bad_content_length async with self.get_dh() as download_handler: - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): response = await download_handler.download_request(request) assert response.text == actual_content_length - log.check_present( - ( - "scrapy.core.http2.stream", - "WARNING", - f"Ignoring bad Content-Length header " - f"{bad_content_length!r} of request {request}, sending " - f"{actual_content_length!r} instead", - ) - ) + assert ( + "scrapy.core.http2.stream", + logging.WARNING, + f"Ignoring bad Content-Length header " + f"{bad_content_length!r} of request {request}, sending " + f"{actual_content_length!r} instead", + ) in caplog.record_tuples @coroutine_test async def test_data_loss_handling(self, mockserver: MockServer) -> None: diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index b4419dc67..7ad103f27 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -2,7 +2,6 @@ import logging from collections.abc import Iterable import pytest -from testfixtures import LogCapture from scrapy.downloadermiddlewares.cookies import CookiesMiddleware from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware @@ -111,14 +110,15 @@ class TestCookiesMiddleware: CookiesMiddleware, ) - def test_setting_enabled_cookies_debug(self): + def test_setting_enabled_cookies_debug( + self, caplog: pytest.LogCaptureFixture + ) -> None: crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True}) mw = CookiesMiddleware.from_crawler(crawler) - with LogCapture( - "scrapy.downloadermiddlewares.cookies", - propagate=False, - level=logging.DEBUG, - ) as log: + caplog.clear() + with caplog.at_level( + logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies" + ): req = Request("http://scrapytest.org/") res = Response( "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} @@ -127,43 +127,44 @@ class TestCookiesMiddleware: req2 = Request("http://scrapytest.org/sub1/") mw.process_request(req2) - log.check( - ( - "scrapy.downloadermiddlewares.cookies", - "DEBUG", - "Received cookies from: <200 http://scrapytest.org/>\n" - "Set-Cookie: C1=value1; path=/\n", - ), - ( - "scrapy.downloadermiddlewares.cookies", - "DEBUG", - "Sending cookies to: \n" - "Cookie: C1=value1\n", - ), - ) + assert caplog.record_tuples == [ + ( + "scrapy.downloadermiddlewares.cookies", + logging.DEBUG, + "Received cookies from: <200 http://scrapytest.org/>\n" + "Set-Cookie: C1=value1; path=/\n", + ), + ( + "scrapy.downloadermiddlewares.cookies", + logging.DEBUG, + "Sending cookies to: \n" + "Cookie: C1=value1\n", + ), + ] - def test_debug_no_cookies(self): + def test_debug_no_cookies(self, caplog: pytest.LogCaptureFixture) -> None: crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True}) mw = CookiesMiddleware.from_crawler(crawler) - with LogCapture( - "scrapy.downloadermiddlewares.cookies", - propagate=False, - level=logging.DEBUG, - ) as log: + caplog.clear() + with caplog.at_level( + logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies" + ): req = Request("http://scrapytest.org/") res = Response("http://scrapytest.org/") # no Set-Cookie header mw.process_response(req, res) mw.process_request(req) # no cookies to send either - log.check() # no log output since cl is empty in both cases + # no log output since cl is empty in both cases + assert caplog.record_tuples == [] - def test_setting_disabled_cookies_debug(self): + def test_setting_disabled_cookies_debug( + self, caplog: pytest.LogCaptureFixture + ) -> None: crawler = get_crawler(settings_dict={"COOKIES_DEBUG": False}) mw = CookiesMiddleware.from_crawler(crawler) - with LogCapture( - "scrapy.downloadermiddlewares.cookies", - propagate=False, - level=logging.DEBUG, - ) as log: + caplog.clear() + with caplog.at_level( + logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies" + ): req = Request("http://scrapytest.org/") res = Response( "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} @@ -172,7 +173,7 @@ class TestCookiesMiddleware: req2 = Request("http://scrapytest.org/sub1/") mw.process_request(req2) - log.check() + assert caplog.record_tuples == [] def test_do_not_break_on_non_utf8_header(self): req = Request("http://scrapytest.org/") @@ -420,44 +421,41 @@ class TestCookiesMiddleware: assert self.mw.process_request(req3) is None self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1") - def test_invalid_cookies(self): + def test_invalid_cookies(self, caplog: pytest.LogCaptureFixture) -> None: """ Invalid cookies are logged as warnings and discarded """ - with LogCapture( - "scrapy.utils.request", - propagate=False, - level=logging.INFO, - ) as lc: + caplog.clear() + with caplog.at_level(logging.INFO, logger="scrapy.utils.request"): cookies1 = [{"value": "bar"}, {"name": "key", "value": "value1"}] - req1 = Request("http://example.org/1", cookies=cookies1) + req1 = Request("http://example.org/1", cookies=cookies1) # type: ignore[arg-type] assert self.mw.process_request(req1) is None cookies2 = [{"name": "foo"}, {"name": "key", "value": "value2"}] - req2 = Request("http://example.org/2", cookies=cookies2) + req2 = Request("http://example.org/2", cookies=cookies2) # type: ignore[arg-type] assert self.mw.process_request(req2) is None cookies3 = [{"name": "foo", "value": None}, {"name": "key", "value": ""}] - req3 = Request("http://example.org/3", cookies=cookies3) + req3 = Request("http://example.org/3", cookies=cookies3) # type: ignore[arg-type] assert self.mw.process_request(req3) is None - lc.check( - ( - "scrapy.utils.request", - "WARNING", - "Invalid cookie found in request :" - " {'value': 'bar', 'secure': False} ('name' is missing)", - ), - ( - "scrapy.utils.request", - "WARNING", - "Invalid cookie found in request :" - " {'name': 'foo', 'secure': False} ('value' is missing)", - ), - ( - "scrapy.utils.request", - "WARNING", - "Invalid cookie found in request :" - " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)", - ), - ) + assert caplog.record_tuples == [ + ( + "scrapy.utils.request", + logging.WARNING, + "Invalid cookie found in request :" + " {'value': 'bar', 'secure': False} ('name' is missing)", + ), + ( + "scrapy.utils.request", + logging.WARNING, + "Invalid cookie found in request :" + " {'name': 'foo', 'secure': False} ('value' is missing)", + ), + ( + "scrapy.utils.request", + logging.WARNING, + "Invalid cookie found in request :" + " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)", + ), + ] self.assertCookieValEqual(req1.headers["Cookie"], "key=value1") self.assertCookieValEqual(req2.headers["Cookie"], "key=value2") self.assertCookieValEqual(req3.headers["Cookie"], "key=") diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index ca18c8038..55f06b396 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -5,7 +5,6 @@ from logging import WARNING from pathlib import Path import pytest -from testfixtures import LogCapture from w3lib.encoding import resolve_encoding from scrapy.downloadermiddlewares.httpcompression import ( @@ -75,7 +74,7 @@ class TestHttpCompression: self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) self.crawler.stats.open_spider() - def _getresponse(self, coding): + def _getresponse(self, coding: str) -> Response: if coding not in FORMAT: raise ValueError @@ -169,29 +168,28 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 74837) - def test_process_response_br_unsupported(self): + def test_process_response_br_unsupported(self, caplog: pytest.LogCaptureFixture): if find_spec("brotli") is not None or find_spec("brotlicffi") is not None: pytest.skip("Requires not having brotli support") response = self._getresponse("br") request = response.request assert response.headers["Content-Encoding"] == b"br" - with LogCapture( - "scrapy.downloadermiddlewares.httpcompression", - propagate=False, - level=WARNING, - ) as log: + caplog.clear() + with caplog.at_level( + WARNING, logger="scrapy.downloadermiddlewares.httpcompression" + ): newresponse = self.mw.process_response(request, response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.downloadermiddlewares.httpcompression", - "WARNING", + WARNING, ( "HttpCompressionMiddleware cannot decode the response for " "http://scrapytest.org/ from unsupported encoding(s) 'br'. " "You need to install brotli or brotlicffi >= 1.2.0 to decode 'br'." ), ), - ) + ] assert newresponse is not response assert newresponse.headers.getlist("Content-Encoding") == [b"br"] @@ -214,29 +212,28 @@ class TestHttpCompression: assert newresponse.body.startswith(b" None: response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = [b"gzip, foo, deflate"] request = response.request - with LogCapture( - "scrapy.downloadermiddlewares.httpcompression", - propagate=False, - level=WARNING, - ) as log: + caplog.clear() + with caplog.at_level( + WARNING, logger="scrapy.downloadermiddlewares.httpcompression" + ): newresponse = self.mw.process_response(request, response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.downloadermiddlewares.httpcompression", - "WARNING", + WARNING, ( "HttpCompressionMiddleware cannot decode the response for" " http://scrapytest.org/ from unsupported encoding(s) 'gzip,foo'." ), ), - ) + ] assert newresponse is not response assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"] @@ -629,7 +627,9 @@ class TestHttpCompression: self._test_compression_bomb_request_meta("zstd") - def _test_download_warnsize_setting(self, compression_id): + def _test_download_warnsize_setting( + self, caplog: pytest.LogCaptureFixture, compression_id: str + ) -> None: settings = {"DOWNLOAD_WARNSIZE": 10_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") @@ -637,41 +637,51 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") - with LogCapture( - "scrapy.downloadermiddlewares.httpcompression", - propagate=False, - level=WARNING, - ) as log: + assert response.request + caplog.clear() + with caplog.at_level( + WARNING, logger="scrapy.downloadermiddlewares.httpcompression" + ): mw.process_response(response.request, response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.downloadermiddlewares.httpcompression", - "WARNING", + WARNING, ( "<200 http://scrapytest.org/> body size after " "decompression (11511612 B) is larger than the download " "warning size (10000000 B)." ), ), - ) + ] - def test_download_warnsize_setting_br(self): + def test_download_warnsize_setting_br( + self, caplog: pytest.LogCaptureFixture + ) -> None: _skip_if_no_br() - self._test_download_warnsize_setting("br") + self._test_download_warnsize_setting(caplog, "br") - def test_download_warnsize_setting_deflate(self): - self._test_download_warnsize_setting("deflate") + def test_download_warnsize_setting_deflate( + self, caplog: pytest.LogCaptureFixture + ) -> None: + self._test_download_warnsize_setting(caplog, "deflate") - def test_download_warnsize_setting_gzip(self): - self._test_download_warnsize_setting("gzip") + def test_download_warnsize_setting_gzip( + self, caplog: pytest.LogCaptureFixture + ) -> None: + self._test_download_warnsize_setting(caplog, "gzip") - def test_download_warnsize_setting_zstd(self): + def test_download_warnsize_setting_zstd( + self, caplog: pytest.LogCaptureFixture + ) -> None: _skip_if_no_zstd() - self._test_download_warnsize_setting("zstd") + self._test_download_warnsize_setting(caplog, "zstd") - def _test_download_warnsize_spider_attr(self, compression_id): + def _test_download_warnsize_spider_attr( + self, caplog: pytest.LogCaptureFixture, compression_id: str + ) -> None: class DownloadWarnSizeSpider(Spider): download_warnsize = 10_000_000 @@ -681,45 +691,55 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") - with LogCapture( - "scrapy.downloadermiddlewares.httpcompression", - propagate=False, - level=WARNING, - ) as log: + assert response.request + caplog.clear() + with caplog.at_level( + WARNING, logger="scrapy.downloadermiddlewares.httpcompression" + ): mw.process_response(response.request, response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.downloadermiddlewares.httpcompression", - "WARNING", + WARNING, ( "<200 http://scrapytest.org/> body size after " "decompression (11511612 B) is larger than the download " "warning size (10000000 B)." ), ), - ) + ] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_download_warnsize_spider_attr_br(self): + def test_download_warnsize_spider_attr_br( + self, caplog: pytest.LogCaptureFixture + ) -> None: _skip_if_no_br() - self._test_download_warnsize_spider_attr("br") + self._test_download_warnsize_spider_attr(caplog, "br") @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_download_warnsize_spider_attr_deflate(self): - self._test_download_warnsize_spider_attr("deflate") + def test_download_warnsize_spider_attr_deflate( + self, caplog: pytest.LogCaptureFixture + ) -> None: + self._test_download_warnsize_spider_attr(caplog, "deflate") @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_download_warnsize_spider_attr_gzip(self): - self._test_download_warnsize_spider_attr("gzip") + def test_download_warnsize_spider_attr_gzip( + self, caplog: pytest.LogCaptureFixture + ) -> None: + self._test_download_warnsize_spider_attr(caplog, "gzip") @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_download_warnsize_spider_attr_zstd(self): + def test_download_warnsize_spider_attr_zstd( + self, caplog: pytest.LogCaptureFixture + ) -> None: _skip_if_no_zstd() - self._test_download_warnsize_spider_attr("zstd") + self._test_download_warnsize_spider_attr(caplog, "zstd") - def _test_download_warnsize_request_meta(self, compression_id): + def _test_download_warnsize_request_meta( + self, caplog: pytest.LogCaptureFixture, compression_id: str + ) -> None: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") mw = HttpCompressionMiddleware.from_crawler(crawler) @@ -727,39 +747,47 @@ class TestHttpCompression: response = self._getresponse(f"bomb-{compression_id}") response.meta["download_warnsize"] = 10_000_000 - with LogCapture( - "scrapy.downloadermiddlewares.httpcompression", - propagate=False, - level=WARNING, - ) as log: + assert response.request + caplog.clear() + with caplog.at_level( + WARNING, logger="scrapy.downloadermiddlewares.httpcompression" + ): mw.process_response(response.request, response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.downloadermiddlewares.httpcompression", - "WARNING", + WARNING, ( "<200 http://scrapytest.org/> body size after " "decompression (11511612 B) is larger than the download " "warning size (10000000 B)." ), ), - ) + ] - def test_download_warnsize_request_meta_br(self): + def test_download_warnsize_request_meta_br( + self, caplog: pytest.LogCaptureFixture + ) -> None: _skip_if_no_br() - self._test_download_warnsize_request_meta("br") + self._test_download_warnsize_request_meta(caplog, "br") - def test_download_warnsize_request_meta_deflate(self): - self._test_download_warnsize_request_meta("deflate") + def test_download_warnsize_request_meta_deflate( + self, caplog: pytest.LogCaptureFixture + ) -> None: + self._test_download_warnsize_request_meta(caplog, "deflate") - def test_download_warnsize_request_meta_gzip(self): - self._test_download_warnsize_request_meta("gzip") + def test_download_warnsize_request_meta_gzip( + self, caplog: pytest.LogCaptureFixture + ) -> None: + self._test_download_warnsize_request_meta(caplog, "gzip") - def test_download_warnsize_request_meta_zstd(self): + def test_download_warnsize_request_meta_zstd( + self, caplog: pytest.LogCaptureFixture + ) -> None: _skip_if_no_zstd() - self._test_download_warnsize_request_meta("zstd") + self._test_download_warnsize_request_meta(caplog, "zstd") def _get_truncated_response(self, compression_id): crawler = get_crawler(Spider) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 825efce2f..410427b84 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,7 +1,9 @@ +from __future__ import annotations + import logging +from typing import Any, cast import pytest -from testfixtures import LogCapture from twisted.internet.error import ConnectError, ConnectionDone, ConnectionLost from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request @@ -85,7 +87,7 @@ class TestRetry: ) assert self.crawler.stats.get_value("retry/count") == 2 - def test_give_up_log_level_setting(self): + def test_give_up_log_level_setting(self, caplog: pytest.LogCaptureFixture) -> None: crawler = get_crawler( DefaultSpider, settings_dict={"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"} ) @@ -94,29 +96,25 @@ class TestRetry: mw.max_retry_times = 0 req = Request("http://example.com/503") rsp = Response("http://example.com/503", body=b"", status=503) - with LogCapture() as log: + with caplog.at_level(logging.WARNING): assert mw.process_response(req, rsp) is rsp - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "WARNING", - f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.WARNING, + f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", + ) in caplog.record_tuples - def test_give_up_log_level_meta(self): + def test_give_up_log_level_meta(self, caplog: pytest.LogCaptureFixture) -> None: self.mw.max_retry_times = 0 req = Request("http://example.com/503", meta={"give_up_log_level": "WARNING"}) rsp = Response("http://example.com/503", body=b"", status=503) - with LogCapture() as log: + with caplog.at_level(logging.WARNING): assert self.mw.process_response(req, rsp) is rsp - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "WARNING", - f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.WARNING, + f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", + ) in caplog.record_tuples def test_twistederrors(self): exceptions = [ @@ -294,14 +292,15 @@ class TestMaxRetryTimes: class TestGetRetryRequest: - def get_spider(self, settings=None): + @staticmethod + def get_spider(settings: dict[str, Any] | None = None) -> Spider: crawler = get_crawler(Spider, settings or {}) return crawler._create_spider("foo") - def test_basic_usage(self): + def test_basic_usage(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): new_request = get_retry_request( request, spider=spider, @@ -313,44 +312,42 @@ class TestGetRetryRequest: assert new_request.meta["retry_times"] == expected_retry_times assert new_request.priority == -1 expected_reason = "unspecified" + assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): assert spider.crawler.stats.get_value(stat) == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_max_retries_reached(self): + def test_max_retries_reached(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() max_retry_times = 0 - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): new_request = get_retry_request( request, spider=spider, max_retry_times=max_retry_times, ) assert new_request is None + assert spider.crawler.stats assert spider.crawler.stats.get_value("retry/max_reached") == 1 failure_count = max_retry_times + 1 expected_reason = "unspecified" - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "ERROR", - f"Gave up retrying {request} (failed {failure_count} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.ERROR, + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_one_retry(self): + def test_one_retry(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): new_request = get_retry_request( request, spider=spider, @@ -363,28 +360,31 @@ class TestGetRetryRequest: assert new_request.meta["retry_times"] == expected_retry_times assert new_request.priority == -1 expected_reason = "unspecified" + assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): assert spider.crawler.stats.get_value(stat) == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_two_retries(self): + def test_two_retries(self, caplog: pytest.LogCaptureFixture) -> None: spider = self.get_spider() request = Request("https://example.com") new_request = request max_retry_times = 2 for index in range(max_retry_times): - with LogCapture() as log: - new_request = get_retry_request( - new_request, - spider=spider, - max_retry_times=max_retry_times, + caplog.clear() + with caplog.at_level(logging.DEBUG): + new_request = cast( + "Request", + get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ), ) assert isinstance(new_request, Request) assert new_request != request @@ -393,36 +393,37 @@ class TestGetRetryRequest: assert new_request.meta["retry_times"] == expected_retry_times assert new_request.priority == -expected_retry_times expected_reason = "unspecified" + assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): value = spider.crawler.stats.get_value(stat) assert value == expected_retry_times - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - with LogCapture() as log: - new_request = get_retry_request( - new_request, - spider=spider, - max_retry_times=max_retry_times, + caplog.clear() + with caplog.at_level(logging.DEBUG): + new_request = cast( + "Request", + get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ), ) assert new_request is None assert spider.crawler.stats.get_value("retry/max_reached") == 1 failure_count = max_retry_times + 1 expected_reason = "unspecified" - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "ERROR", - f"Gave up retrying {request} (failed {failure_count} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.ERROR, + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) in caplog.record_tuples def test_no_spider(self): request = Request("https://example.com") @@ -483,238 +484,231 @@ class TestGetRetryRequest: ) assert new_request.priority == priority_adjust - def test_log_extra_retry_success(self): + def test_log_extra_retry_success(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture(attributes=("spider",)) as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, ) - log.check_present(spider) + assert any(getattr(r, "spider", None) is spider for r in caplog.records) - def test_log_extra_retries_exceeded(self): + def test_log_extra_retries_exceeded(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture(attributes=("spider",)) as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, max_retry_times=0, ) - log.check_present(spider) + assert any(getattr(r, "spider", None) is spider for r in caplog.records) - def test_reason_string(self): + def test_reason_string(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() expected_reason = "because" - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, reason=expected_reason, ) expected_retry_times = 1 + assert spider.crawler.stats for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): assert spider.crawler.stats.get_value(stat) == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_reason_builtin_exception(self): + def test_reason_builtin_exception(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() expected_reason = NotImplementedError() expected_reason_string = "builtins.NotImplementedError" - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, reason=expected_reason, ) expected_retry_times = 1 + assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) assert stat == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_reason_builtin_exception_class(self): + def test_reason_builtin_exception_class( + self, caplog: pytest.LogCaptureFixture + ) -> None: request = Request("https://example.com") spider = self.get_spider() expected_reason = NotImplementedError expected_reason_string = "builtins.NotImplementedError" - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, reason=expected_reason, ) expected_retry_times = 1 + assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) assert stat == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_reason_custom_exception(self): + def test_reason_custom_exception(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() expected_reason = IgnoreRequest() expected_reason_string = "scrapy.exceptions.IgnoreRequest" - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, reason=expected_reason, ) expected_retry_times = 1 + assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) assert stat == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_reason_custom_exception_class(self): + def test_reason_custom_exception_class( + self, caplog: pytest.LogCaptureFixture + ) -> None: request = Request("https://example.com") spider = self.get_spider() expected_reason = IgnoreRequest expected_reason_string = "scrapy.exceptions.IgnoreRequest" - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, reason=expected_reason, ) expected_retry_times = 1 + assert spider.crawler.stats stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) assert stat == 1 - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "DEBUG", - f"Retrying {request} (failed {expected_retry_times} times): " - f"{expected_reason}", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) in caplog.record_tuples - def test_custom_logger(self): + def test_custom_logger(self, caplog: pytest.LogCaptureFixture) -> None: logger = logging.getLogger("custom-logger") request = Request("https://example.com") spider = self.get_spider() expected_reason = "because" - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): get_retry_request( request, spider=spider, reason=expected_reason, logger=logger, ) - log.check_present( - ( - "custom-logger", - "DEBUG", - f"Retrying {request} (failed 1 times): {expected_reason}", - ) - ) + assert ( + "custom-logger", + logging.DEBUG, + f"Retrying {request} (failed 1 times): {expected_reason}", + ) in caplog.record_tuples - def test_give_up_log_level_default(self): + def test_give_up_log_level_default(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture() as log: + with caplog.at_level(logging.ERROR): get_retry_request( request, spider=spider, max_retry_times=0, ) - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "ERROR", - f"Gave up retrying {request} (failed 1 times): unspecified", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.ERROR, + f"Gave up retrying {request} (failed 1 times): unspecified", + ) in caplog.record_tuples - def test_give_up_log_level_argument_name(self): + def test_give_up_log_level_argument_name( + self, caplog: pytest.LogCaptureFixture + ) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture() as log: + with caplog.at_level(logging.WARNING): get_retry_request( request, spider=spider, max_retry_times=0, give_up_log_level="WARNING", ) - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "WARNING", - f"Gave up retrying {request} (failed 1 times): unspecified", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.WARNING, + f"Gave up retrying {request} (failed 1 times): unspecified", + ) in caplog.record_tuples - def test_give_up_log_level_argument_number(self): + def test_give_up_log_level_argument_number( + self, caplog: pytest.LogCaptureFixture + ) -> None: request = Request("https://example.com") spider = self.get_spider() - with LogCapture() as log: + with caplog.at_level(logging.WARNING): get_retry_request( request, spider=spider, max_retry_times=0, give_up_log_level=logging.WARNING, ) - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "WARNING", - f"Gave up retrying {request} (failed 1 times): unspecified", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.WARNING, + f"Gave up retrying {request} (failed 1 times): unspecified", + ) in caplog.record_tuples - def test_give_up_log_level_setting(self): + def test_give_up_log_level_setting(self, caplog: pytest.LogCaptureFixture) -> None: request = Request("https://example.com") spider = self.get_spider({"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"}) - with LogCapture() as log: + with caplog.at_level(logging.WARNING): get_retry_request( request, spider=spider, max_retry_times=0, ) - log.check_present( - ( - "scrapy.downloadermiddlewares.retry", - "WARNING", - f"Gave up retrying {request} (failed 1 times): unspecified", - ) - ) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.WARNING, + f"Gave up retrying {request} (failed 1 times): unspecified", + ) in caplog.record_tuples def test_give_up_log_level_invalid(self): request = Request("https://example.com") diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 5d79691b2..479332eae 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -1,11 +1,14 @@ +from __future__ import annotations + import hashlib +import logging import shutil import sys import tempfile from pathlib import Path +from typing import TYPE_CHECKING, Any import pytest -from testfixtures import LogCapture from scrapy.core.scheduler import Scheduler from scrapy.dupefilters import BaseDupeFilter, RFPDupeFilter @@ -15,8 +18,16 @@ from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider +if TYPE_CHECKING: + from scrapy.crawler import Crawler -def _get_dupefilter(*, crawler=None, settings=None, open_=True): + +def _get_dupefilter( + *, + crawler: Crawler | None = None, + settings: dict[str, Any] | None = None, + open_: bool = True, +) -> BaseDupeFilter: if crawler is None: crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) @@ -151,108 +162,71 @@ class TestRFPDupeFilter: finally: shutil.rmtree(path) - def test_log(self): - with LogCapture() as log: - settings = { - "DUPEFILTER_DEBUG": False, - "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, - } - crawler = get_crawler(SimpleSpider, settings_dict=settings) - spider = SimpleSpider.from_crawler(crawler) - dupefilter = _get_dupefilter(crawler=crawler) + def test_log(self, caplog: pytest.LogCaptureFixture) -> None: + settings = { + "DUPEFILTER_DEBUG": False, + "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, + } + crawler = get_crawler(SimpleSpider, settings_dict=settings) + spider = SimpleSpider.from_crawler(crawler) + dupefilter = _get_dupefilter(crawler=crawler) - r1 = Request("http://scrapytest.org/index.html") - r2 = Request("http://scrapytest.org/index.html") + r1 = Request("http://scrapytest.org/index.html") + r2 = Request("http://scrapytest.org/index.html") + with caplog.at_level(logging.DEBUG): dupefilter.log(r1, spider) dupefilter.log(r2, spider) - assert crawler.stats.get_value("dupefilter/filtered") == 2 - log.check_present( - ( - "scrapy.dupefilters", - "DEBUG", - "Filtered duplicate request: - no more" - " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)", - ) - ) + assert crawler.stats + assert crawler.stats.get_value("dupefilter/filtered") == 2 + assert ( + "scrapy.dupefilters", + logging.DEBUG, + "Filtered duplicate request: - no more" + " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)", + ) in caplog.record_tuples - dupefilter.close("finished") + dupefilter.close("finished") - def test_log_debug(self): - with LogCapture() as log: - settings = { - "DUPEFILTER_DEBUG": True, - "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, - } - crawler = get_crawler(SimpleSpider, settings_dict=settings) - spider = SimpleSpider.from_crawler(crawler) - dupefilter = _get_dupefilter(crawler=crawler) + @pytest.mark.parametrize("df", [None, FromCrawlerRFPDupeFilter]) + def test_log_debug( + self, caplog: pytest.LogCaptureFixture, df: type[BaseDupeFilter] | None + ) -> None: + settings: dict[str, Any] = { + "DUPEFILTER_DEBUG": True, + } + if df: + settings["DUPEFILTER_CLASS"] = df + crawler = get_crawler(SimpleSpider, settings_dict=settings) + spider = SimpleSpider.from_crawler(crawler) + dupefilter = _get_dupefilter(crawler=crawler) - r1 = Request("http://scrapytest.org/index.html") - r2 = Request( - "http://scrapytest.org/index.html", - headers={"Referer": "http://scrapytest.org/INDEX.html"}, - ) + r1 = Request("http://scrapytest.org/index.html") + r2 = Request( + "http://scrapytest.org/index.html", + headers={"Referer": "http://scrapytest.org/INDEX.html"}, + ) + with caplog.at_level(logging.DEBUG): dupefilter.log(r1, spider) dupefilter.log(r2, spider) - assert crawler.stats.get_value("dupefilter/filtered") == 2 - log.check_present( - ( - "scrapy.dupefilters", - "DEBUG", - "Filtered duplicate request: (referer: None)", - ) - ) - log.check_present( - ( - "scrapy.dupefilters", - "DEBUG", - "Filtered duplicate request: " - " (referer: http://scrapytest.org/INDEX.html)", - ) - ) + assert crawler.stats + assert crawler.stats.get_value("dupefilter/filtered") == 2 + assert ( + "scrapy.dupefilters", + logging.DEBUG, + "Filtered duplicate request: (referer: None)", + ) in caplog.record_tuples + assert ( + "scrapy.dupefilters", + logging.DEBUG, + "Filtered duplicate request: " + " (referer: http://scrapytest.org/INDEX.html)", + ) in caplog.record_tuples - dupefilter.close("finished") - - def test_log_debug_default_dupefilter(self): - with LogCapture() as log: - settings = { - "DUPEFILTER_DEBUG": True, - } - crawler = get_crawler(SimpleSpider, settings_dict=settings) - spider = SimpleSpider.from_crawler(crawler) - dupefilter = _get_dupefilter(crawler=crawler) - - r1 = Request("http://scrapytest.org/index.html") - r2 = Request( - "http://scrapytest.org/index.html", - headers={"Referer": "http://scrapytest.org/INDEX.html"}, - ) - - dupefilter.log(r1, spider) - dupefilter.log(r2, spider) - - assert crawler.stats.get_value("dupefilter/filtered") == 2 - log.check_present( - ( - "scrapy.dupefilters", - "DEBUG", - "Filtered duplicate request: (referer: None)", - ) - ) - log.check_present( - ( - "scrapy.dupefilters", - "DEBUG", - "Filtered duplicate request: " - " (referer: http://scrapytest.org/INDEX.html)", - ) - ) - - dupefilter.close("finished") + dupefilter.close("finished") class TestBaseDupeFilter: diff --git a/tests/test_engine.py b/tests/test_engine.py index 87f0265a0..8a3bceccb 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,13 +1,13 @@ from __future__ import annotations import asyncio +import logging import subprocess import sys from typing import TYPE_CHECKING, Any from unittest.mock import Mock import pytest -from testfixtures import LogCapture from scrapy import signals from scrapy.core.engine import ExecutionEngine, _Slot @@ -145,8 +145,10 @@ class TestEngine(TestEngineBase): await asyncio.gather(e.start_async(), e.start_async()) await e.stop_async() - @inline_callbacks_test - def test_start_request_processing_exception(self): + @coroutine_test + async def test_start_request_processing_exception( + self, caplog: pytest.LogCaptureFixture + ) -> None: class BadRequestFingerprinter: def fingerprint(self, request): raise ValueError # to make Scheduler.enqueue_request() fail @@ -160,10 +162,10 @@ class TestEngine(TestEngineBase): crawler = get_crawler( SimpleSpider, {"REQUEST_FINGERPRINTER_CLASS": BadRequestFingerprinter} ) - with LogCapture() as log: - yield crawler.crawl() - assert "Error while processing requests from start()" in str(log) - assert "Spider closed (shutdown)" in str(log) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async() + assert "Error while processing requests from start()" in caplog.text + assert "Spider closed (shutdown)" in caplog.text def test_short_timeout(self): args = ( diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f1c416fbd..92c414bef 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2,6 +2,7 @@ from __future__ import annotations import csv import json +import logging import marshal import pickle import tempfile @@ -12,7 +13,6 @@ from unittest import mock import lxml.etree import pytest -from testfixtures import LogCapture from w3lib.url import file_uri_to_path import scrapy @@ -563,7 +563,9 @@ class TestFeedExport(TestFeedExportBase): assert expctd == data[fmt] @coroutine_test - async def test_export_no_items_multiple_feeds(self): + async def test_export_no_items_multiple_feeds( + self, caplog: pytest.LogCaptureFixture + ) -> None: """Make sure that `storage.store` is not called.""" settings = { "FEEDS": { @@ -575,10 +577,10 @@ class TestFeedExport(TestFeedExportBase): "FEED_STORE_EMPTY": False, } - with LogCapture() as log: + with caplog.at_level(logging.INFO): await self.exported_no_data(settings) - assert str(log).count("Storage.store is called") == 0 + assert caplog.text.count("Storage.store is called") == 0 @coroutine_test async def test_export_multiple_item_classes(self): @@ -1080,7 +1082,9 @@ class TestFeedExport(TestFeedExportBase): assert data["csv"] == b"" @coroutine_test - async def test_multiple_feeds_success_logs_blocking_feed_storage(self): + async def test_multiple_feeds_success_logs_blocking_feed_storage( + self, caplog: pytest.LogCaptureFixture + ): settings = { "FEEDS": { self._random_temp_filename(): {"format": "json"}, @@ -1093,14 +1097,16 @@ class TestFeedExport(TestFeedExportBase): {"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}, ] - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): await self.exported_data(items, settings) for fmt in ["json", "xml", "csv"]: - assert f"Stored {fmt} feed (2 items)" in str(log) + assert f"Stored {fmt} feed (2 items)" in caplog.text @coroutine_test - async def test_multiple_feeds_failing_logs_blocking_feed_storage(self): + async def test_multiple_feeds_failing_logs_blocking_feed_storage( + self, caplog: pytest.LogCaptureFixture + ): settings = { "FEEDS": { self._random_temp_filename(): {"format": "json"}, @@ -1113,11 +1119,11 @@ class TestFeedExport(TestFeedExportBase): {"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}, ] - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): await self.exported_data(items, settings) for fmt in ["json", "xml", "csv"]: - assert f"Error storing {fmt} feed (2 items)" in str(log) + assert f"Error storing {fmt} feed (2 items)" in caplog.text @coroutine_test async def test_extend_kwargs(self): diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index bdd0d7614..66488540f 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import os import string import tempfile @@ -10,7 +11,6 @@ from unittest import mock from urllib.parse import quote import pytest -from testfixtures import LogCapture from w3lib.url import path_to_file_uri import scrapy @@ -415,23 +415,21 @@ class TestS3FeedStorage: acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] assert acl == "custom-acl" - def test_overwrite_default(self): - with LogCapture() as log: - S3FeedStorage( - "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" - ) - assert "S3 does not support appending to files" not in str(log) + def test_overwrite_default(self, caplog: pytest.LogCaptureFixture) -> None: + S3FeedStorage( + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" + ) + assert "S3 does not support appending to files" not in caplog.text - def test_overwrite_false(self): - with LogCapture() as log: - S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - "custom-acl", - feed_options={"overwrite": False}, - ) - assert "S3 does not support appending to files" in str(log) + def test_overwrite_false(self, caplog: pytest.LogCaptureFixture) -> None: + S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + "custom-acl", + feed_options={"overwrite": False}, + ) + assert "S3 does not support appending to files" in caplog.text class TestGCSFeedStorage: @@ -505,20 +503,20 @@ class TestGCSFeedStorage: blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) f.close.assert_called_once_with() - def test_overwrite_default(self): - with LogCapture() as log: + def test_overwrite_default(self, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.DEBUG): GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl") - assert "GCS does not support appending to files" not in str(log) + assert "GCS does not support appending to files" not in caplog.text - def test_overwrite_false(self): - with LogCapture() as log: + def test_overwrite_false(self, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.DEBUG): GCSFeedStorage( "gs://mybucket/export.csv", "myproject-123", "custom-acl", feed_options={"overwrite": False}, ) - assert "GCS does not support appending to files" in str(log) + assert "GCS does not support appending to files" in caplog.text class TestStdoutFeedStorage: @@ -530,17 +528,18 @@ class TestStdoutFeedStorage: storage.store(file) assert out.getvalue() == b"content" - def test_overwrite_default(self): - with LogCapture() as log: + def test_overwrite_default(self, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.DEBUG): StdoutFeedStorage("stdout:") assert ( "Standard output (stdout) storage does not support overwriting" - not in str(log) + not in caplog.text ) - def test_overwrite_true(self): - with LogCapture() as log: + def test_overwrite_true(self, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.DEBUG): StdoutFeedStorage("stdout:", feed_options={"overwrite": True}) - assert "Standard output (stdout) storage does not support overwriting" in str( - log + assert ( + "Standard output (stdout) storage does not support overwriting" + in caplog.text ) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 360aa613e..fc6ccf981 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -1,7 +1,9 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING import pytest -from testfixtures import LogCapture from twisted.python.failure import Failure from scrapy.exceptions import DropItem @@ -10,9 +12,11 @@ from scrapy.item import Field, Item from scrapy.logformatter import LogFormatter from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.spiders import ItemSpider -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer class CustomItem(Item): @@ -254,15 +258,6 @@ class DropSomeItemsPipeline: class TestShowOrSkipMessages: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - def setup_method(self): self.base_settings = { "LOG_LEVEL": "DEBUG", @@ -271,22 +266,26 @@ class TestShowOrSkipMessages: }, } - @inline_callbacks_test - def test_show_messages(self): + @coroutine_test + async def test_show_messages( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(ItemSpider, self.base_settings) - with LogCapture() as lc: - yield crawler.crawl(mockserver=self.mockserver) - assert "Scraped from <200 http://127.0.0.1:" in str(lc) - assert "Crawled (200) None: settings = self.base_settings.copy() settings["LOG_FORMATTER"] = SkipMessagesLogFormatter crawler = get_crawler(ItemSpider, settings) - with LogCapture() as lc: - yield crawler.crawl(mockserver=self.mockserver) - assert "Scraped from <200 http://127.0.0.1:" not in str(lc) - assert "Crawled (200) None: assert len(items) == 1 assert self.media_key in items[0] @@ -125,13 +127,16 @@ class TestFileDownloadCrawl: for i in item[self.media_key]: assert (self.tmpmediastore / i["path"]).exists() - def _assert_files_download_failure(self, crawler, items, code, logs): + def _assert_files_download_failure( + self, crawler: Crawler, items: list[Any], code: int, logs: str + ) -> None: # check that the item does NOT have the "images/files" field populated assert len(items) == 1 assert self.media_key in items[0] assert not items[0][self.media_key] # check that there was 1 successful fetch and 3 other responses with non-200 code + assert crawler.stats assert crawler.stats.get_value("downloader/request_method_count/GET") == 4 assert crawler.stats.get_value("downloader/response_count") == 4 assert crawler.stats.get_value("downloader/response_status_count/200") == 1 @@ -144,62 +149,71 @@ class TestFileDownloadCrawl: # check that no files were written to the media store assert not list(self.tmpmediastore.iterdir()) - @inline_callbacks_test - def test_download_media(self): + @coroutine_test + async def test_download_media(self, caplog: pytest.LogCaptureFixture) -> None: crawler = self._create_crawler(MediaDownloadSpider) - with LogCapture() as log: - yield crawler.crawl( + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, ) - self._assert_files_downloaded(self.items, str(log)) + self._assert_files_downloaded(self.items, caplog.text) - @inline_callbacks_test - def test_download_media_wrong_urls(self): + @coroutine_test + async def test_download_media_wrong_urls( + self, caplog: pytest.LogCaptureFixture + ) -> None: crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) - with LogCapture() as log: - yield crawler.crawl( + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, ) - self._assert_files_download_failure(crawler, self.items, 404, str(log)) + self._assert_files_download_failure(crawler, self.items, 404, caplog.text) - @inline_callbacks_test - def test_download_media_redirected_default_failure(self): + @coroutine_test + async def test_download_media_redirected_default_failure( + self, caplog: pytest.LogCaptureFixture + ): crawler = self._create_crawler(RedirectedMediaDownloadSpider) - with LogCapture() as log: - yield crawler.crawl( + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver, ) - self._assert_files_download_failure(crawler, self.items, 302, str(log)) + self._assert_files_download_failure(crawler, self.items, 302, caplog.text) - @inline_callbacks_test - def test_download_media_redirected_allowed(self): + @coroutine_test + async def test_download_media_redirected_allowed( + self, caplog: pytest.LogCaptureFixture + ) -> None: settings = { **self.settings, "MEDIA_ALLOW_REDIRECTS": True, } crawler = self._create_crawler(RedirectedMediaDownloadSpider, settings) - with LogCapture() as log: - yield crawler.crawl( + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver, ) - self._assert_files_downloaded(self.items, str(log)) + self._assert_files_downloaded(self.items, caplog.text) + assert crawler.stats assert crawler.stats.get_value("downloader/response_status_count/302") == 3 - @inline_callbacks_test - def test_download_media_file_path_error(self): + @coroutine_test + async def test_download_media_file_path_error( + self, caplog: pytest.LogCaptureFixture + ) -> None: cls = load_object(self.pipeline_class) - class ExceptionRaisingMediaPipeline(cls): + class ExceptionRaisingMediaPipeline(cls): # type: ignore[misc,valid-type] def file_path(self, request, response=None, info=None, *, item=None): return 1 / 0 @@ -208,14 +222,14 @@ class TestFileDownloadCrawl: "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1}, } crawler = self._create_crawler(MediaDownloadSpider, settings) - with LogCapture() as log: - yield crawler.crawl( + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( self.mockserver.url("/static/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver, ) - assert "ZeroDivisionError" in str(log) + assert "ZeroDivisionError" in caplog.text pillow_available: bool diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index be471e5fe..23c19e4be 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -4,7 +4,6 @@ import logging from unittest.mock import MagicMock import pytest -from testfixtures import LogCapture from twisted.python.failure import Failure from scrapy import signals @@ -129,7 +128,7 @@ class TestBaseMediaPipeline: context = getattr(info.downloaded[fp].value, "__context__", None) assert context is None - def test_default_item_completed(self): + def test_default_item_completed(self, caplog: pytest.LogCaptureFixture) -> None: item = {"name": "name"} assert self.pipe.item_completed([], item, self.info) is item @@ -137,21 +136,20 @@ class TestBaseMediaPipeline: fail = Failure(Exception()) results = [(True, 1), (False, fail)] - with LogCapture() as log: - new_item = self.pipe.item_completed(results, item, self.info) - + caplog.clear() + new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item - assert len(log.records) == 1 - record = log.records[0] + assert len(caplog.records) == 1 + record = caplog.records[0] assert record.levelname == "ERROR" assert record.exc_info == failure_to_exc_info(fail) # disable failure logging and check again + caplog.clear() self.pipe.LOG_FAILED_RESULTS = False - with LogCapture() as log: - new_item = self.pipe.item_completed(results, item, self.info) + new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item - assert len(log.records) == 0 + assert len(caplog.records) == 0 def test_item_completed_filtered_request_not_logged( self, caplog: pytest.LogCaptureFixture diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 38d56e9bd..a624d2097 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -1,11 +1,17 @@ -from testfixtures import LogCapture +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING from scrapy import Request, signals from scrapy.http.response import Response from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import SingleRequestSpider -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test, inline_callbacks_test + +if TYPE_CHECKING: + import pytest OVERRIDDEN_URL = "https://example.org" @@ -63,6 +69,8 @@ class AlternativeCallbacksMiddleware: class TestCrawl: + mockserver: MockServer + @classmethod def setup_class(cls): cls.mockserver = MockServer() @@ -107,8 +115,10 @@ class TestCrawl: assert failure.request.url == url assert isinstance(failure.value, ZeroDivisionError) - @inline_callbacks_test - def test_downloader_middleware_override_request_in_process_response(self): + @coroutine_test + async def test_downloader_middleware_override_request_in_process_response( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ Downloader middleware which returns a response with an specific 'request' attribute. @@ -133,22 +143,21 @@ class TestCrawl: ) crawler.signals.connect(signal_handler, signal=signals.response_received) - with LogCapture() as log: - yield crawler.crawl(seed=url, mockserver=self.mockserver) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.request.url == OVERRIDDEN_URL assert signal_params["response"].url == url assert signal_params["request"].url == OVERRIDDEN_URL - log.check_present( - ( - "scrapy.core.engine", - "DEBUG", - f"Crawled (200) (referer: None)", - ), - ) + assert ( + "scrapy.core.engine", + logging.DEBUG, + f"Crawled (200) (referer: None)", + ) in caplog.record_tuples @inline_callbacks_test def test_downloader_middleware_override_in_process_exception(self): @@ -196,8 +205,10 @@ class TestCrawl: assert response.body == b"Caught ZeroDivisionError" assert response.request.url == url - @inline_callbacks_test - def test_downloader_middleware_alternative_callback(self): + @coroutine_test + async def test_downloader_middleware_alternative_callback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ Downloader middleware which returns a response with a specific 'request' attribute, with an alternative callback @@ -211,14 +222,11 @@ class TestCrawl: }, ) - with LogCapture() as log: - url = self.mockserver.url("/status?n=200") - yield crawler.crawl(seed=url, mockserver=self.mockserver) - - log.check_present( - ( - "alternative_callbacks_spider", - "INFO", - "alt_callback was invoked with foo=bar", - ), - ) + url = self.mockserver.url("/status?n=200") + with caplog.at_level(logging.INFO): + await crawler.crawl_async(seed=url, mockserver=self.mockserver) + assert ( + "alternative_callbacks_spider", + logging.INFO, + "alt_callback was invoked with foo=bar", + ) in caplog.record_tuples diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index ee9d2ed51..b88893b2b 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,10 +1,17 @@ -from testfixtures import LogCapture +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING from scrapy.http import Request from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.spiders import MockServerSpider -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + import pytest + + from tests.mockserver.http import MockServer class InjectArgumentsDownloaderMiddleware: @@ -147,33 +154,32 @@ class KeywordArgumentsSpider(MockServerSpider): class TestCallbackKeywordArguments: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - @inline_callbacks_test - def test_callback_kwargs(self): + @coroutine_test + async def test_callback_kwargs( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(KeywordArgumentsSpider) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.ERROR): + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, KeywordArgumentsSpider) assert all(crawler.spider.checks) + assert crawler.stats assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks") # check exceptions for argument mismatch exceptions = {} - for line in log.records: + for line in caplog.records: for key in ("takes_less", "takes_more"): if key in line.getMessage(): exceptions[key] = line - assert exceptions["takes_less"].exc_info[0] is TypeError - assert str(exceptions["takes_less"].exc_info[1]).endswith( + takes_less_exc_info = exceptions["takes_less"].exc_info + assert takes_less_exc_info is not None + assert takes_less_exc_info[0] is TypeError + assert str(takes_less_exc_info[1]).endswith( "parse_takes_less() got an unexpected keyword argument 'number'" - ), "Exception message: " + str(exceptions["takes_less"].exc_info[1]) - assert exceptions["takes_more"].exc_info[0] is TypeError - assert str(exceptions["takes_more"].exc_info[1]).endswith( + ) + takes_more_exc_info = exceptions["takes_more"].exc_info + assert takes_more_exc_info is not None + assert takes_more_exc_info[0] is TypeError + assert str(takes_more_exc_info[1]).endswith( "parse_takes_more() missing 1 required positional argument: 'other'" - ), "Exception message: " + str(exceptions["takes_more"].exc_info[1]) + ) diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 08acacae7..2c2d26d53 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -1,9 +1,10 @@ from __future__ import annotations +import logging +from typing import TYPE_CHECKING from urllib.parse import urljoin import pytest -from testfixtures import LogCapture from twisted.internet import defer from scrapy.core.scheduler import BaseScheduler @@ -12,8 +13,10 @@ from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.request import fingerprint from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test, inline_callbacks_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer PATHS = ["/a", "/b", "/c"] URLS = [urljoin("https://example.org", p) for p in PATHS] @@ -147,18 +150,19 @@ class TestSimpleScheduler(InterfaceCheckMixin): class TestMinimalSchedulerCrawl: scheduler_cls = MinimalScheduler - @inline_callbacks_test - def test_crawl(self): - with MockServer() as mockserver: - settings = { - "SCHEDULER": self.scheduler_cls, - } - with LogCapture() as log: - crawler = get_crawler(PathsSpider, settings) - yield crawler.crawl(mockserver) - for path in PATHS: - assert f"{{'path': '{path}'}}" in str(log) - assert f"'item_scraped_count': {len(PATHS)}" in str(log) + @coroutine_test + async def test_crawl( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + settings = { + "SCHEDULER": self.scheduler_cls, + } + with caplog.at_level(logging.DEBUG): + crawler = get_crawler(PathsSpider, settings) + await crawler.crawl_async(mockserver) + for path in PATHS: + assert f"{{'path': '{path}'}}" in caplog.text + assert f"'item_scraped_count': {len(PATHS)}" in caplog.text class TestSimpleSchedulerCrawl(TestMinimalSchedulerCrawl): diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py index c8cfd364e..fd62e0016 100644 --- a/tests/test_spider_sitemap.py +++ b/tests/test_spider_sitemap.py @@ -8,7 +8,6 @@ from logging import WARNING from pathlib import Path import pytest -from testfixtures import LogCapture from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse from scrapy.spiders import SitemapSpider @@ -254,22 +253,23 @@ Sitemap: /sitemap-relative-url.xml urls = [req.url for req in spider._parse_sitemap(r)] assert urls == result - def test_parse_sitemap_empty_body(self): + def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None: r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"") spider = self.spider_class("example.com") - with LogCapture() as lc: + caplog.clear() + with caplog.at_level(WARNING): results = list(spider._parse_sitemap(r)) assert not results - lc.check( + assert caplog.record_tuples == [ ( "scrapy.spiders.sitemap", - "WARNING", + WARNING, "Ignoring invalid sitemap: <200 http://www.example.com/sitemap.xml>", ) - ) + ] def test_parse_sitemap_not_sitemap(self): body = b""" @@ -342,7 +342,7 @@ Sitemap: /sitemap-relative-url.xml response = Response(url="https://example.com", body=body, request=request) assert spider._get_sitemap_body(response) is None - def test_download_warnsize_setting(self): + def test_download_warnsize_setting(self, caplog: pytest.LogCaptureFixture) -> None: settings = {"DOWNLOAD_WARNSIZE": 10_000_000} crawler = get_crawler(settings_dict=settings) spider = self.spider_class.from_crawler(crawler, "example.com") @@ -350,25 +350,26 @@ Sitemap: /sitemap-relative-url.xml body = body_path.read_bytes() request = Request(url="https://example.com") response = Response(url="https://example.com", body=body, request=request) - with LogCapture( - "scrapy.spiders.sitemap", propagate=False, level=WARNING - ) as log: + caplog.clear() + with caplog.at_level(WARNING, logger="scrapy.spiders.sitemap"): spider._get_sitemap_body(response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.spiders.sitemap", - "WARNING", + WARNING, ( "<200 https://example.com> body size after decompression " "(11511612 B) is larger than the download warning size " "(10000000 B)." ), ), - ) + ] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_download_warnsize_spider_attr(self): - class DownloadWarnSizeSpider(self.spider_class): + def test_download_warnsize_spider_attr( + self, caplog: pytest.LogCaptureFixture + ) -> None: + class DownloadWarnSizeSpider(self.spider_class): # type: ignore[name-defined,misc] download_warnsize = 10_000_000 crawler = get_crawler() @@ -379,23 +380,24 @@ Sitemap: /sitemap-relative-url.xml url="https://example.com", meta={"download_warnsize": 10_000_000} ) response = Response(url="https://example.com", body=body, request=request) - with LogCapture( - "scrapy.spiders.sitemap", propagate=False, level=WARNING - ) as log: + caplog.clear() + with caplog.at_level(WARNING, logger="scrapy.spiders.sitemap"): spider._get_sitemap_body(response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.spiders.sitemap", - "WARNING", + WARNING, ( "<200 https://example.com> body size after decompression " "(11511612 B) is larger than the download warning size " "(10000000 B)." ), ), - ) + ] - def test_download_warnsize_request_meta(self): + def test_download_warnsize_request_meta( + self, caplog: pytest.LogCaptureFixture + ) -> None: crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") @@ -404,21 +406,20 @@ Sitemap: /sitemap-relative-url.xml url="https://example.com", meta={"download_warnsize": 10_000_000} ) response = Response(url="https://example.com", body=body, request=request) - with LogCapture( - "scrapy.spiders.sitemap", propagate=False, level=WARNING - ) as log: + caplog.clear() + with caplog.at_level(WARNING, logger="scrapy.spiders.sitemap"): spider._get_sitemap_body(response) - log.check( + assert caplog.record_tuples == [ ( "scrapy.spiders.sitemap", - "WARNING", + WARNING, ( "<200 https://example.com> body size after decompression " "(11511612 B) is larger than the download warning size " "(10000000 B)." ), ), - ) + ] @coroutine_test async def test_sitemap_urls(self): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index afdfaa322..dacc90b27 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -1,10 +1,16 @@ -from testfixtures import LogCapture +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING from scrapy import Request, Spider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + import pytest + class _BaseSpiderMiddleware: def __init__(self, crawler): @@ -250,113 +256,132 @@ class TestSpiderMiddleware: def teardown_class(cls): cls.mockserver.__exit__(None, None, None) - async def crawl_log(self, spider: type[Spider]) -> LogCapture: + async def crawl_log( + self, spider: type[Spider], caplog: pytest.LogCaptureFixture + ) -> str: crawler = get_crawler(spider) - with LogCapture() as log: + caplog.clear() + with caplog.at_level(logging.DEBUG): await crawler.crawl_async(mockserver=self.mockserver) - return log + return caplog.text @coroutine_test - async def test_recovery(self): + async def test_recovery(self, caplog: pytest.LogCaptureFixture) -> None: """ (0) Recover from an exception in a spider callback. The final item count should be 3 (one yielded from the callback method before the exception is raised, one directly from the recovery middleware and one from the spider when processing the request that was enqueued from the recovery middleware) """ - log = await self.crawl_log(RecoverySpider) - assert "Middleware: TabError exception caught" in str(log) - assert str(log).count("Middleware: TabError exception caught") == 1 - assert "'item_scraped_count': 3" in str(log) + log = await self.crawl_log(RecoverySpider, caplog) + assert "Middleware: TabError exception caught" in log + assert log.count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in log @coroutine_test - async def test_recovery_asyncgen(self): + async def test_recovery_asyncgen(self, caplog: pytest.LogCaptureFixture) -> None: """ Same as test_recovery but with an async callback. """ - log = await self.crawl_log(RecoveryAsyncGenSpider) - assert "Middleware: TabError exception caught" in str(log) - assert str(log).count("Middleware: TabError exception caught") == 1 - assert "'item_scraped_count': 3" in str(log) + log = await self.crawl_log(RecoveryAsyncGenSpider, caplog) + assert "Middleware: TabError exception caught" in log + assert log.count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in log @coroutine_test - async def test_process_spider_input_without_errback(self): + async def test_process_spider_input_without_errback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ (1.1) An exception from the process_spider_input chain should be caught by the process_spider_exception chain from the start if the Request has no errback """ - log1 = await self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) - assert "Middleware: will raise IndexError" in str(log1) - assert "Middleware: IndexError exception caught" in str(log1) + log1 = await self.crawl_log(ProcessSpiderInputSpiderWithoutErrback, caplog) + assert "Middleware: will raise IndexError" in log1 + assert "Middleware: IndexError exception caught" in log1 @coroutine_test - async def test_process_spider_input_with_errback(self): + async def test_process_spider_input_with_errback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ (1.2) An exception from the process_spider_input chain should not be caught by the process_spider_exception chain if the Request has an errback """ - log1 = await self.crawl_log(ProcessSpiderInputSpiderWithErrback) - assert "Middleware: IndexError exception caught" not in str(log1) - assert "Middleware: will raise IndexError" in str(log1) - assert "Got a Failure on the Request errback" in str(log1) - assert "{'from': 'errback'}" in str(log1) - assert "{'from': 'callback'}" not in str(log1) - assert "'item_scraped_count': 1" in str(log1) + log1 = await self.crawl_log(ProcessSpiderInputSpiderWithErrback, caplog) + assert "Middleware: IndexError exception caught" not in log1 + assert "Middleware: will raise IndexError" in log1 + assert "Got a Failure on the Request errback" in log1 + assert "{'from': 'errback'}" in log1 + assert "{'from': 'callback'}" not in log1 + assert "'item_scraped_count': 1" in log1 @coroutine_test - async def test_generator_callback(self): + async def test_generator_callback(self, caplog: pytest.LogCaptureFixture) -> None: """ (2) An exception from a spider callback (returning a generator) should be caught by the process_spider_exception chain. Items yielded before the exception is raised should be processed normally. """ - log2 = await self.crawl_log(GeneratorCallbackSpider) - assert "Middleware: ImportError exception caught" in str(log2) - assert "'item_scraped_count': 2" in str(log2) + log2 = await self.crawl_log(GeneratorCallbackSpider, caplog) + assert "Middleware: ImportError exception caught" in log2 + assert "'item_scraped_count': 2" in log2 @coroutine_test - async def test_async_generator_callback(self): + async def test_async_generator_callback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ Same as test_generator_callback but with an async callback. """ - log2 = await self.crawl_log(AsyncGeneratorCallbackSpider) - assert "Middleware: ImportError exception caught" in str(log2) - assert "'item_scraped_count': 2" in str(log2) + log2 = await self.crawl_log(AsyncGeneratorCallbackSpider, caplog) + assert "Middleware: ImportError exception caught" in log2 + assert "'item_scraped_count': 2" in log2 @coroutine_test - async def test_generator_callback_right_after_callback(self): + async def test_generator_callback_right_after_callback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ (2.1) Special case of (2): Exceptions should be caught even if the middleware is placed right after the spider """ - log21 = await self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider) - assert "Middleware: ImportError exception caught" in str(log21) - assert "'item_scraped_count': 2" in str(log21) + log21 = await self.crawl_log( + GeneratorCallbackSpiderMiddlewareRightAfterSpider, caplog + ) + assert "Middleware: ImportError exception caught" in log21 + assert "'item_scraped_count': 2" in log21 @coroutine_test - async def test_not_a_generator_callback(self): + async def test_not_a_generator_callback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ (3) An exception from a spider callback (returning a list) should be caught by the process_spider_exception chain. No items should be processed. """ - log3 = await self.crawl_log(NotGeneratorCallbackSpider) - assert "Middleware: ZeroDivisionError exception caught" in str(log3) - assert "item_scraped_count" not in str(log3) + log3 = await self.crawl_log(NotGeneratorCallbackSpider, caplog) + assert "Middleware: ZeroDivisionError exception caught" in log3 + assert "item_scraped_count" not in log3 @coroutine_test - async def test_not_a_generator_callback_right_after_callback(self): + async def test_not_a_generator_callback_right_after_callback( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ (3.1) Special case of (3): Exceptions should be caught even if the middleware is placed right after the spider """ log31 = await self.crawl_log( - NotGeneratorCallbackSpiderMiddlewareRightAfterSpider + NotGeneratorCallbackSpiderMiddlewareRightAfterSpider, caplog ) - assert "Middleware: ZeroDivisionError exception caught" in str(log31) - assert "item_scraped_count" not in str(log31) + assert "Middleware: ZeroDivisionError exception caught" in log31 + assert "item_scraped_count" not in log31 @coroutine_test - async def test_generator_output_chain(self): + async def test_generator_output_chain( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ (4) An exception from a middleware's process_spider_output method should be sent to the process_spider_exception method from the next middleware in the chain. @@ -365,23 +390,23 @@ class TestSpiderMiddleware: The final item count should be 2 (one from the spider callback and one from the process_spider_exception chain) """ - log4 = await self.crawl_log(GeneratorOutputChainSpider) - assert "'item_scraped_count': 2" in str(log4) + log4 = await self.crawl_log(GeneratorOutputChainSpider, caplog) + assert "'item_scraped_count': 2" in log4 assert ( "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught" - in str(log4) + in log4 ) assert ( "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught" - in str(log4) + in log4 ) assert ( "GeneratorFailMiddleware.process_spider_exception: LookupError caught" - not in str(log4) + not in log4 ) assert ( "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught" - not in str(log4) + not in log4 ) item_from_callback = { "processed": [ @@ -398,6 +423,6 @@ class TestSpiderMiddleware: "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", ] } - assert str(item_from_callback) in str(log4) - assert str(item_recovered) in str(log4) - assert "parse-second-item" not in str(log4) + assert str(item_from_callback) in log4 + assert str(item_recovered) in log4 + assert "parse-second-item" not in log4 diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index fe57492b6..7f5301387 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -8,7 +8,6 @@ from io import StringIO from typing import TYPE_CHECKING, Any import pytest -from testfixtures import LogCapture from twisted.python.failure import Failure from scrapy.utils.log import ( @@ -107,15 +106,15 @@ class TestLogCounterHandler: class TestStreamLogger: - def test_redirect(self): + def test_redirect(self, caplog: pytest.LogCaptureFixture) -> None: logger = logging.getLogger("test") logger.setLevel(logging.WARNING) old_stdout = sys.stdout sys.stdout = StreamLogger(logger, logging.ERROR) - with LogCapture() as log: - print("test log msg") - log.check(("test", "ERROR", "test log msg")) + caplog.clear() + print("test log msg") + assert caplog.record_tuples == [("test", logging.ERROR, "test log msg")] sys.stdout = old_stdout diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index dc615eca6..dbb9caf2a 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,23 +1,25 @@ from __future__ import annotations import asyncio -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import pytest from pydispatch import dispatcher -from testfixtures import LogCapture from twisted.internet import defer from twisted.python.failure import Failure from scrapy.utils.asyncio import call_later -from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.signal import ( send_catch_log, send_catch_log_async, send_catch_log_deferred, ) from scrapy.utils.test import get_from_asyncio_queue -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from collections.abc import Callable if TYPE_CHECKING: from collections.abc import Callable @@ -27,25 +29,22 @@ class TestSendCatchLog: # whether the function being tested returns exceptions or failures returns_exceptions: bool = False - @inline_callbacks_test - def test_send_catch_log(self): + @coroutine_test + async def test_send_catch_log(self, caplog: pytest.LogCaptureFixture) -> None: test_signal = object() handlers_called: set[Callable[..., None]] = set() dispatcher.connect(self.error_handler, signal=test_signal) dispatcher.connect(self.ok_handler, signal=test_signal) - with LogCapture() as log: - result = yield defer.maybeDeferred( - self._get_result, - test_signal, - arg="test", - handlers_called=handlers_called, - ) + caplog.clear() + result = await ensure_awaitable( + self._get_result(test_signal, arg="test", handlers_called=handlers_called) + ) assert self.error_handler in handlers_called assert self.ok_handler in handlers_called - assert len(log.records) == 1 - record = log.records[0] + assert len(caplog.records) == 1 + record = caplog.records[0] assert "error_handler" in record.getMessage() assert record.levelname == "ERROR" assert result[0][0] == self.error_handler # pylint: disable=comparison-with-callable @@ -57,7 +56,7 @@ class TestSendCatchLog: dispatcher.disconnect(self.error_handler, signal=test_signal) dispatcher.disconnect(self.ok_handler, signal=test_signal) - def _get_result(self, signal, *a, **kw): + def _get_result(self, signal: Any, *a: Any, **kw: Any) -> Any: return send_catch_log(signal, *a, **kw) def error_handler(self, arg, handlers_called): @@ -72,7 +71,7 @@ class TestSendCatchLog: @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestSendCatchLogDeferred(TestSendCatchLog): - def _get_result(self, signal, *a, **kw): + def _get_result(self, signal: Any, *a: Any, **kw: Any) -> Any: return send_catch_log_deferred(signal, *a, **kw) @@ -137,14 +136,15 @@ class TestSendCatchLogAsyncAsyncio(TestSendCatchLogAsync): class TestSendCatchLog2: - def test_error_logged_if_deferred_not_supported(self): + def test_error_logged_if_deferred_not_supported( + self, caplog: pytest.LogCaptureFixture + ) -> None: def test_handler(): return defer.Deferred() test_signal = object() dispatcher.connect(test_handler, test_signal) - with LogCapture() as log: - send_catch_log(test_signal) - assert len(log.records) == 1 - assert "Cannot return deferreds from signal handler" in str(log) + send_catch_log(test_signal) + assert len(caplog.records) == 1 + assert "Cannot return deferreds from signal handler" in caplog.text dispatcher.disconnect(test_handler, test_signal) diff --git a/tests/utils/bases/spider.py b/tests/utils/bases/spider.py index f774fde07..799c1820d 100644 --- a/tests/utils/bases/spider.py +++ b/tests/utils/bases/spider.py @@ -1,11 +1,11 @@ from __future__ import annotations +import logging from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any from unittest import mock import pytest -from testfixtures import LogCapture from scrapy import signals from scrapy.crawler import Crawler @@ -109,15 +109,15 @@ class TestSpiderBase(ABC): yield crawler.crawl() assert crawler.settings.get("TEST1") == "spider_instance" - def test_logger(self): + def test_logger(self, caplog: pytest.LogCaptureFixture) -> None: spider = self.spider_class("example.com") - with LogCapture() as lc: + caplog.clear() + with caplog.at_level(logging.INFO): spider.logger.info("test log msg") - lc.check(("example.com", "INFO", "test log msg")) + assert caplog.record_tuples == [("example.com", logging.INFO, "test log msg")] - record = lc.records[0] - assert "spider" in record.__dict__ - assert record.spider is spider + record = caplog.records[0] + assert getattr(record, "spider", None) is spider def test_log(self): spider = self.spider_class("example.com") diff --git a/tox.ini b/tox.ini index d35fe91db..e10a7cc0c 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,6 @@ deps = pytest-cov >= 7.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 - testfixtures pytest-twisted >= 1.14.3 [testenv] From 59ebb26e60ea1e776ed11d00aef1d20bdb535e98 Mon Sep 17 00:00:00 2001 From: Shadow_Lu Date: Tue, 28 Jul 2026 15:51:55 +0530 Subject: [PATCH 013/111] Fix CaseInsensitiveDict.copy() sharing state with the original (#7783) * Fix CaseInsensitiveDict.copy() sharing state with the original * Address review: don't re-normalise in __copy__, keep _keys in sync in __ior__ UserDict.__ior__ writes self.data directly, bypassing __setitem__, so _keys never learned about the new keys. --- scrapy/utils/datatypes.py | 16 ++++++++++++++++ tests/test_utils_datatypes.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index c020ff4b9..e761a2474 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -132,6 +132,22 @@ class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]): def __repr__(self) -> str: return f"<{self.__class__.__name__}: {super().__repr__()}>" + # UserDict.copy() shallow-copies the instance, which would share self._keys + # between the copy and the original. + def __copy__(self) -> Self: + new = self.__class__() + new.data = self.data.copy() + new._keys = self._keys.copy() + return new + + copy = __copy__ + + # UserDict.__ior__ updates self.data directly, which would leave self._keys + # out of date. + def __ior__(self, other: Any) -> Self: # type: ignore[override,misc] + self.update(other) + return self + def _normkey(self, key: str | bytes) -> str | bytes: return key diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index fe1f60c7d..f43d20e69 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -204,6 +204,15 @@ class TestCaseInsensitiveDictBase(ABC): assert h1.get("header1") == h3.get("header1") assert h1.get("header1") == h3.get("HEADER1") + def test_copy_is_independent(self): + h1 = self.dict_class({"header1": "value1", "header2": "value2"}) + for h2 in (copy.copy(h1), h1.copy()): + del h2["header1"] + h2["header3"] = "value3" + assert "header1" in h1 + assert "header3" not in h1 + assert dict(h1) == {"header1": "value1", "header2": "value2"} + class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): dict_class = CaseInsensitiveDict # type: ignore[assignment] @@ -220,6 +229,28 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): assert isinstance(iterkeys, Iterator) assert list(iterkeys) == ["AsDf", "FoO"] + def test_copy_keeps_values(self): + class MyDict(self.dict_class): + def _normvalue(self, value): + return value + 1 + + d = MyDict({"key": 1}) + for copied in (copy.copy(d), d.copy()): + assert copied["key"] == 2 + + def test_ior(self): + d = self.dict_class({"header1": "value1"}) + d |= {"HEADER1": "value2", "header2": "value3"} + assert len(d) == 2 + assert d["HeAdEr1"] == "value2" + assert d["HeAdEr2"] == "value3" + + def test_ior_mapping(self): + d = self.dict_class({"header1": "value1"}) + d |= self.dict_class({"HEADER1": "value2"}) + assert len(d) == 1 + assert d["HeAdEr1"] == "value2" + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestCaselessDict(TestCaseInsensitiveDictBase): From 5a65bdcc18e51a85fbc9eca0a25e4be7aa4ce4e3 Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 28 Jul 2026 12:24:04 +0200 Subject: [PATCH 014/111] Improve spiders coverage (#7768) * Skip the deprecated scrapy.mail in coverage data * Improve CrawlSpider coverage * Improve XMLFeedSpider coverage * Improve SitemapSpider coverage * Solve mypy issues * Align new spider tests with the shared test helper structure --- scrapy/mail.py | 2 + scrapy/spiders/feed.py | 11 +-- tests/spiders.py | 46 +++++++++++++ tests/test_crawl.py | 16 +++++ tests/test_spider.py | 130 ++++++++++++++++++++++++++++++++++- tests/test_spider_crawl.py | 43 ++++++++++++ tests/test_spider_sitemap.py | 62 +++++++++++++++++ tests/utils/crawl.py | 27 ++++++++ 8 files changed, 326 insertions(+), 11 deletions(-) create mode 100644 tests/utils/crawl.py diff --git a/scrapy/mail.py b/scrapy/mail.py index 97123e63c..0691312a3 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -2,6 +2,8 @@ Mail sending helpers """ +# pragma: no file cover + from __future__ import annotations import logging diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 925f31ede..1e7ac9c34 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -9,7 +9,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any -from scrapy.exceptions import NotConfigured, NotSupported +from scrapy.exceptions import NotSupported from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.spiders import Spider @@ -76,11 +76,6 @@ class XMLFeedSpider(Spider): yield from self.process_results(response, ret) def _parse(self, response: Response, **kwargs: Any) -> Any: - if not hasattr(self, "parse_node"): - raise NotConfigured( - "You must define parse_node method in order to scrape this XML feed" - ) - response = self.adapt_response(response) nodes: Iterable[Selector] if self.iterator == "iternodes": @@ -158,9 +153,5 @@ class CSVFeedSpider(Spider): yield from self.process_results(response, ret) def _parse(self, response: Response, **kwargs: Any) -> Any: - if not hasattr(self, "parse_row"): - raise NotConfigured( - "You must define parse_row method in order to scrape this CSV feed" - ) response = self.adapt_response(response) return self.parse_rows(response) diff --git a/tests/spiders.py b/tests/spiders.py index da14fdbe3..7c7d3007c 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -38,6 +38,35 @@ class MockServerSpider(Spider): self.is_secure = is_secure +class RawResponseSpider(MockServerSpider): + """Base class for spiders that fetch a response built by the test itself. + + Subclasses return the body from :meth:`raw_body` and request + :attr:`raw_url`, which the mock server answers with that body verbatim + under :attr:`content_type`. This lets tests reach parsing code that only + a specific kind of response triggers while still going through a regular + crawl, instead of calling internal parsing methods directly. + """ + + name = "raw_response" + content_type = "text/plain" + + def raw_body(self) -> str: + raise NotImplementedError + + @property + def raw_url(self) -> str: + assert self.mockserver + raw = ( + "HTTP/1.1 200 OK\r\n" + f"Content-Type: {self.content_type}\r\n" + "Connection: close\r\n" + "\r\n" + f"{self.raw_body()}" + ) + return self.mockserver.url("/raw?" + urlencode({"raw": raw})) + + class MetaSpider(MockServerSpider): name = "meta" @@ -496,6 +525,23 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): self.logger.info("[errback] status %i", failure.value.response.status) +class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod): + name = "crawl_spider_without_errback" + + async def start(self): + test_body = b""" + + Page title + +

Item 200

+

Item 404

+ + + """ + url = self.mockserver.url("/alpayload") + yield Request(url, method="POST", body=test_body) + + class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod): name = "crawl_spider_with_process_request_cb_kwargs" rules = ( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 9c479068a..d284805be 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -41,6 +41,7 @@ from tests.spiders import ( CrawlSpiderWithAsyncCallback, CrawlSpiderWithAsyncGeneratorCallback, CrawlSpiderWithErrback, + CrawlSpiderWithoutErrback, CrawlSpiderWithParseMethod, CrawlSpiderWithProcessRequestCallbackKeywordArguments, DelaySpider, @@ -505,6 +506,21 @@ class TestCrawlSpider: assert "[errback] status 500" in caplog.text assert "[errback] status 501" in caplog.text + @coroutine_test + async def test_crawlspider_without_errback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + crawler = get_crawler(CrawlSpiderWithoutErrback) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) + + # The failing request (404) is followed by a rule without an errback, + # so the failure is dropped silently and the crawl finishes normally. + assert "[parse] status 200 (foo: None)" in caplog.text + assert "[errback]" not in caplog.text + assert crawler.stats + assert crawler.stats.get_value("downloader/response_status_count/404") == 1 + @coroutine_test async def test_crawlspider_process_request_cb_kwargs( self, caplog: pytest.LogCaptureFixture, mockserver: MockServer diff --git a/tests/test_spider.py b/tests/test_spider.py index 03d17199f..38cb8da18 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,11 +1,26 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import pytest -from scrapy.http import Response, TextResponse, XmlResponse +from scrapy.http import Request, Response, TextResponse, XmlResponse from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider from tests import get_testdata +from tests.spiders import RawResponseSpider from tests.utils.bases.spider import TestSpiderBase +from tests.utils.crawl import crawl_items +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer + + +class RawFeedSpider(RawResponseSpider): + content_type = "text/xml" + + async def start(self): + yield Request(self.raw_url) class TestSpider(TestSpiderBase): @@ -60,6 +75,89 @@ class TestXMLFeedSpider(TestSpiderBase): }, ], iterator + @coroutine_test + async def test_parse_node_uses_parse_item(self, mockserver: MockServer): + # parse_node falls back to parse_item for backward compatibility. + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + itertag = "item" + + def raw_body(self): + return "1" + + def parse_item(self, response, selector): + return {"id": selector.xpath("id/text()").get()} + + items, _ = await crawl_items(_Spider, mockserver) + assert items == [{"id": "1"}] + + @coroutine_test + async def test_parse_node_not_defined(self, mockserver: MockServer): + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + itertag = "item" + + def raw_body(self): + return "1" + + items, crawler = await crawl_items(_Spider, mockserver) + assert items == [] + assert crawler.stats + assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1 + + @coroutine_test + async def test_html_iterator(self, mockserver: MockServer): + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + iterator = "html" + itertag = "item" + content_type = "text/html" + + def raw_body(self): + return ( + "1" + "2" + ) + + def parse_node(self, response, selector): + return {"id": selector.xpath("id/text()").get()} + + items, _ = await crawl_items(_Spider, mockserver) + assert items == [{"id": "1"}, {"id": "2"}] + + @coroutine_test + async def test_unsupported_iterator(self, mockserver: MockServer): + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + iterator = "unsupported" + + def raw_body(self): + return "" + + def parse_node(self, response, selector): + return {} + + items, crawler = await crawl_items(_Spider, mockserver) + assert items == [] + assert crawler.stats + assert crawler.stats.get_value("spider_exceptions/NotSupported") == 1 + + @pytest.mark.parametrize("feed_iterator", ["xml", "html"]) + @coroutine_test + async def test_non_text_response(self, feed_iterator: str, mockserver: MockServer): + # The xml and html iterators require a text response. + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + content_type = "application/octet-stream" + iterator = feed_iterator + + def raw_body(self): + # A binary (non-text) body, so the response is a plain Response. + return "\x00\x01\x02\x03" + + def parse_node(self, response, selector): + return {} + + items, crawler = await crawl_items(_Spider, mockserver) + assert items == [] + assert crawler.stats + assert crawler.stats.get_value("spider_exceptions/ValueError") == 1 + class TestCSVFeedSpider(TestSpiderBase): spider_class = CSVFeedSpider @@ -81,6 +179,36 @@ class TestCSVFeedSpider(TestSpiderBase): assert rows[0] == {"id": "1", "name": "alpha", "value": "foobar"} assert len(rows) == 4 + @coroutine_test + async def test_parse(self, mockserver: MockServer): + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + content_type = "text/csv" + delimiter = "," + quotechar = "'" + + def raw_body(self): + return get_testdata("feeds", "feed-sample6.csv").decode() + + def parse_row(self, response, row): + return row + + items, _ = await crawl_items(_Spider, mockserver) + assert items[0] == {"id": "1", "name": "alpha", "value": "foobar"} + assert len(items) == 4 + + @coroutine_test + async def test_parse_row_not_defined(self, mockserver: MockServer): + class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc] + content_type = "text/csv" + + def raw_body(self): + return "id\n1\n" + + items, crawler = await crawl_items(_Spider, mockserver) + assert items == [] + assert crawler.stats + assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1 + class TestNoParseMethodSpider: spider_class = Spider diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py index f34f9add9..9d5f0548b 100644 --- a/tests/test_spider_crawl.py +++ b/tests/test_spider_crawl.py @@ -12,6 +12,7 @@ from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule, Spider from scrapy.utils.test import get_crawler from tests.utils.bases.spider import TestSpiderBase +from tests.utils.decorators import coroutine_test class TestCrawlSpider(TestSpiderBase): @@ -293,6 +294,48 @@ class TestCrawlSpider(TestSpiderBase): TextResponse(spider.start_urls, body=b""), None, None ) + @coroutine_test + async def test_parse_with_rules_without_callback(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(CrawlSpider): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(),) + + spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider)) + results = [ + r async for r in spider.parse_with_rules(response, None, {}, follow=True) + ] + assert [r.url for r in results] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + + @coroutine_test + async def test_parse_with_rules_without_following(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + item = {"name": "item"} + + class _CrawlSpider(CrawlSpider): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(),) + + spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider)) + results = [ + r + async for r in spider.parse_with_rules( + response, lambda response: [item], {}, follow=False + ) + ] + assert results == [item] + class TestDeprecation: def test_crawl_spider(self): diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py index fd62e0016..2f1ccab81 100644 --- a/tests/test_spider_sitemap.py +++ b/tests/test_spider_sitemap.py @@ -6,6 +6,7 @@ from datetime import datetime from io import BytesIO from logging import WARNING from pathlib import Path +from typing import TYPE_CHECKING import pytest @@ -13,9 +14,30 @@ from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlRespon from scrapy.spiders import SitemapSpider from scrapy.utils.test import get_crawler from tests import tests_datadir +from tests.spiders import RawResponseSpider from tests.utils.bases.spider import TestSpiderBase +from tests.utils.crawl import crawl_items from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from tests.mockserver.http import MockServer + + +class RawSitemapSpider(RawResponseSpider): + """Feeds :meth:`raw_body` to :class:`~scrapy.spiders.SitemapSpider` as a + sitemap, so that it is fetched and followed through a regular crawl. + + Subclasses build the document in :meth:`raw_body`, typically using + :attr:`mockserver` to point ```` entries at real endpoints. + """ + + content_type = "application/xml" + + async def start(self): + self.sitemap_urls = [self.raw_url] + async for request in super().start(): + yield request + class TestSitemapSpider(TestSpiderBase): spider_class = SitemapSpider @@ -253,6 +275,46 @@ Sitemap: /sitemap-relative-url.xml urls = [req.url for req in spider._parse_sitemap(r)] assert urls == result + @coroutine_test + async def test_sitemap_rules_with_callable(self, mockserver: MockServer): + # A sitemap_rules entry may hold a callable instead of a method name. + def parse_item(response): + yield {"url": response.url} + + class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc] + sitemap_rules = [("", parse_item)] + + def raw_body(self): + loc = self.mockserver.url("/text") + return ( + '' + '' + f"{loc}" + "" + ) + + items, _ = await crawl_items(_Spider, mockserver) + assert items == [{"url": mockserver.url("/text")}] + + @coroutine_test + async def test_sitemap_empty_loc(self, mockserver: MockServer): + class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc] + def parse(self, response): + yield {"url": response.url} + + def raw_body(self): + loc = self.mockserver.url("/text") + return ( + '' + '' + "" + f"{loc}" + "" + ) + + items, _ = await crawl_items(_Spider, mockserver) + assert items == [{"url": mockserver.url("/text")}] + def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None: r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"") spider = self.spider_class("example.com") diff --git a/tests/utils/crawl.py b/tests/utils/crawl.py new file mode 100644 index 000000000..4631d909d --- /dev/null +++ b/tests/utils/crawl.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from scrapy import signals +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + from scrapy.spiders import Spider + from tests.mockserver.http import MockServer + + +async def crawl_items( + spider_cls: type[Spider], mockserver: MockServer, **kwargs: Any +) -> tuple[list[Any], Crawler]: + """Run *spider_cls* against *mockserver* and return the scraped items along + with the crawler, which gives tests access to the resulting stats.""" + items: list[Any] = [] + + def collect(item: Any) -> None: + items.append(item) + + crawler = get_crawler(spider_cls) + crawler.signals.connect(collect, signals.item_scraped) + await crawler.crawl_async(mockserver=mockserver, **kwargs) + return items, crawler From ad816d2b3a00c04c86ac8b3fd85e19a7e1b355f8 Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 28 Jul 2026 14:04:54 +0200 Subject: [PATCH 015/111] Improve test coverage for scrapy.cmdline (#7795) --- scrapy/cmdline.py | 6 +- tests/test_commands.py | 244 ++++++++++++++++++++++++++++++---- tests/utils/bases/commands.py | 6 + 3 files changed, 226 insertions(+), 30 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 6c306afdb..e6d5ff96a 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -225,13 +225,11 @@ def _run_command(cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace) def _run_command_profiled( cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace ) -> None: - if opts.profile: - sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") + sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") loc = locals() p = cProfile.Profile() p.runctx("cmd.run(args, opts)", globals(), loc) - if opts.profile: - p.dump_stats(opts.profile) + p.dump_stats(opts.profile) if __name__ == "__main__": diff --git a/tests/test_commands.py b/tests/test_commands.py index 51f98db1b..3e687e811 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,14 +3,12 @@ from __future__ import annotations import argparse import json import sys -from io import StringIO from typing import TYPE_CHECKING -from unittest import mock import pytest import scrapy -from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg +from scrapy.cmdline import _pop_command_name, execute from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings @@ -153,12 +151,6 @@ class MySpider(scrapy.Spider): self._append_settings(proj_mod_path, "LOG_LEVEL = 'DEBUG'\n") - @staticmethod - def _append_settings(proj_mod_path: Path, text: str) -> None: - """Add text to the end of the project settings.py.""" - with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f: - f.write(text) - @staticmethod def _replace_custom_settings( proj_mod_path: Path, spider_name: str, text: str @@ -347,23 +339,223 @@ class TestMiscCommands(TestProjectBase): subdir.mkdir(exist_ok=True) assert call("list", cwd=subdir) == 0 - def test_command_not_found(self) -> None: - na_msg = """ -The list command is not available from this location. -These commands are only available from within a project: check, crawl, edit, list, parse. -""" - not_found_msg = """ -Unknown command: abc -""" - params = [ - ("list", False, na_msg), - ("abc", False, not_found_msg), - ("abc", True, not_found_msg), - ] - for cmdname, inproject, message in params: - with mock.patch("sys.stdout", new=StringIO()) as out: - _print_unknown_command_msg(Settings(), cmdname, inproject) - assert out.getvalue().strip() == message.strip() + +class TestCommandListing(TestProjectBase): + """Tests for the command list that ``scrapy`` prints when called without a + command name.""" + + def test_outside_project(self) -> None: + returncode, out, err = proc() + assert returncode == 0, err + assert f"Scrapy {scrapy.__version__} - no active project" in out + assert "Available commands:" in out + assert "Create new project" in out + assert "More commands available when run from project directory" in out + assert 'Use "scrapy -h" to see more info about a command' in out + + def test_inside_project(self, proj_path: Path) -> None: + returncode, out, err = proc(cwd=proj_path) + assert returncode == 0, err + assert ( + f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out + ) + assert "List available spiders" in out + assert "More commands available when run from project directory" not in out + + +class TestUnknownCommand(TestProjectBase): + def test_outside_project(self) -> None: + returncode, out, err = proc("abc") + assert returncode == 2, err + assert f"Scrapy {scrapy.__version__} - no active project" in out + assert "Unknown command: abc" in out + assert 'Use "scrapy" to see available commands' in out + + def test_inside_project(self, proj_path: Path) -> None: + returncode, out, err = proc("abc", cwd=proj_path) + assert returncode == 2, err + assert ( + f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out + ) + assert "Unknown command: abc" in out + + def test_project_only_command_outside_project(self) -> None: + returncode, out, err = proc("list") + assert returncode == 2, err + assert "The list command is not available from this location." in out + assert ( + "These commands are only available from within a project: " + "check, crawl, edit, list, parse." in out + ) + + +class TestCommandsModule(TestProjectBase): + """Tests for commands defined in the module of the COMMANDS_MODULE setting.""" + + @pytest.fixture + def proj_path_with_commands(self, proj_path: Path) -> Path: + commands_path = proj_path / self.project_name / "commands" + commands_path.mkdir() + (commands_path / "__init__.py").touch() + (commands_path / "mycmd.py").write_text( + """ +from scrapy.commands import ScrapyCommand + + +class Command(ScrapyCommand): + requires_crawler_process = False + + def short_desc(self): + return "My custom command" + + def run(self, args, opts): + print("My custom command ran") +""", + encoding="utf-8", + ) + (commands_path / "helpcmd.py").write_text( + """ +from scrapy.commands import ScrapyCommand +from scrapy.exceptions import UsageError + + +class Command(ScrapyCommand): + requires_crawler_process = False + + def short_desc(self): + return "My command that asks for its help message" + + def run(self, args, opts): + raise UsageError +""", + encoding="utf-8", + ) + (commands_path / "silentcmd.py").write_text( + """ +from scrapy.commands import ScrapyCommand +from scrapy.exceptions import UsageError + + +class Command(ScrapyCommand): + requires_crawler_process = False + + def short_desc(self): + return "My command that fails silently" + + def run(self, args, opts): + raise UsageError(print_help=False) +""", + encoding="utf-8", + ) + self._append_settings( + proj_path / self.project_name, + f'\nCOMMANDS_MODULE = "{self.project_name}.commands"\n', + ) + return proj_path + + def test_listed(self, proj_path_with_commands: Path) -> None: + returncode, out, err = proc(cwd=proj_path_with_commands) + assert returncode == 0, err + assert "My custom command" in out + + def test_run(self, proj_path_with_commands: Path) -> None: + returncode, out, err = proc("mycmd", cwd=proj_path_with_commands) + assert returncode == 0, err + assert "My custom command ran" in out + + def test_usage_error(self, proj_path_with_commands: Path) -> None: + """A message-less UsageError makes the help message be printed.""" + returncode, out, err = proc("helpcmd", cwd=proj_path_with_commands) + assert returncode == 2, err + assert "scrapy helpcmd" in out + + def test_usage_error_without_help(self, proj_path_with_commands: Path) -> None: + """A message-less UsageError with print_help disabled prints nothing.""" + returncode, out, err = proc("silentcmd", cwd=proj_path_with_commands) + assert returncode == 2, err + assert not out + + +class TestEntryPointCommands: + """Tests for commands defined in the scrapy.commands entry point group.""" + + @staticmethod + def _write_dist(path: Path, entry_point: str) -> None: + """Write into *path* a package with a command and a function, and the + metadata of an installed distribution that declares *entry_point* in + the scrapy.commands entry point group. + + Since ``python -m scrapy.cmdline`` puts the current working directory + in the import path, running it with *path* as the working directory + makes Scrapy find that entry point. + """ + package_path = path / "mycmds" + package_path.mkdir() + (package_path / "__init__.py").touch() + (package_path / "mycmd.py").write_text( + """ +from scrapy.commands import ScrapyCommand + + +class Command(ScrapyCommand): + requires_crawler_process = False + + def short_desc(self): + return "My entry point command" + + def run(self, args, opts): + print("My entry point command ran") + + +def not_a_command(): + pass +""", + encoding="utf-8", + ) + dist_info_path = path / "mycmds-1.0.dist-info" + dist_info_path.mkdir() + (dist_info_path / "METADATA").write_text( + "Metadata-Version: 2.1\nName: mycmds\nVersion: 1.0\n", encoding="utf-8" + ) + (dist_info_path / "entry_points.txt").write_text( + f"[scrapy.commands]\n{entry_point}\n", encoding="utf-8" + ) + + def test_listed(self, tmp_path: Path) -> None: + self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command") + returncode, out, err = proc(cwd=tmp_path) + assert returncode == 0, err + assert "My entry point command" in out + + def test_run(self, tmp_path: Path) -> None: + self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command") + returncode, out, err = proc("mycmd", cwd=tmp_path) + assert returncode == 0, err + assert "My entry point command ran" in out + + def test_not_a_class(self, tmp_path: Path) -> None: + self._write_dist(tmp_path, "mycmd = mycmds.mycmd:not_a_command") + returncode, _, err = proc("version", cwd=tmp_path) + assert returncode == 1 + assert "ValueError: Invalid entry point mycmd" in err + + +class TestExecute: + """Tests for calls to scrapy.cmdline.execute() from Python code, which the + command line does not cover.""" + + def test_argv(self, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit) as exc_info: + execute(["scrapy", "version"]) + assert exc_info.value.code == 0 + assert scrapy.__version__ in capsys.readouterr().out + + def test_settings(self, capsys: pytest.CaptureFixture[str]) -> None: + settings = Settings() + with pytest.raises(SystemExit) as exc_info: + execute(["scrapy", "settings", "--get", "BOT_NAME"], settings=settings) + assert exc_info.value.code == 0 + assert capsys.readouterr().out.strip() == "scrapybot" class TestBenchCommand: diff --git a/tests/utils/bases/commands.py b/tests/utils/bases/commands.py index 594544c83..55ef686fa 100644 --- a/tests/utils/bases/commands.py +++ b/tests/utils/bases/commands.py @@ -32,3 +32,9 @@ class TestProjectBase: proj_path = tmp_path / self.project_name copytree(_proj_path_cached, proj_path) return proj_path + + @staticmethod + def _append_settings(proj_mod_path: Path, text: str) -> None: + """Add text to the end of the project settings.py.""" + with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f: + f.write(text) From e7d8b34e73ef2598b27ea0bca31a292e008fff76 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 Jul 2026 17:24:34 +0500 Subject: [PATCH 016/111] Next refactoring pass of test_utils_*. (#7797) --- pyproject.toml | 7 - scrapy/core/downloader/handlers/ftp.py | 3 +- scrapy/http/headers.py | 26 +- scrapy/http/request/__init__.py | 12 +- scrapy/http/response/__init__.py | 23 +- scrapy/http/response/text.py | 12 +- scrapy/utils/datatypes.py | 40 +- scrapy/utils/decorators.py | 17 +- tests/test_utils_asyncgen.py | 24 +- tests/test_utils_asyncio.py | 9 +- tests/test_utils_curl.py | 2 + tests/test_utils_datatypes.py | 60 +-- tests/test_utils_decorators.py | 20 +- tests/test_utils_defer.py | 24 +- tests/test_utils_deprecate.py | 46 +- tests/test_utils_display.py | 16 +- tests/test_utils_gz.py | 2 + tests/test_utils_httpobj.py | 2 + tests/test_utils_misc/__init__.py | 261 +++++------ ...t_return_with_argument_inside_generator.py | 421 +++++++++--------- tests/test_utils_project.py | 10 +- tests/test_utils_python.py | 21 +- tests/test_utils_reactor.py | 2 + tests/test_utils_request.py | 26 +- tests/test_utils_response.py | 6 +- tests/test_utils_serialize.py | 2 + tests/test_utils_signal.py | 3 - tests/test_utils_sitemap.py | 2 + tests/test_utils_template.py | 9 +- tests/test_utils_trackref.py | 6 +- tests/test_utils_url.py | 4 +- tests_typing/test_http_request.mypy-testing | 2 +- tests_typing/test_http_response.mypy-testing | 2 +- 33 files changed, 619 insertions(+), 503 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9b1e64121..576a42e5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -178,13 +178,6 @@ module = [ "tests.test_squeues", "tests.test_squeues_request", "tests.test_stats", - "tests.test_utils_datatypes", - "tests.test_utils_decorators", - "tests.test_utils_defer", - "tests.test_utils_deprecate", - "tests.test_utils_misc.test_return_with_argument_inside_generator", - "tests.test_utils_python", - "tests.test_utils_request", "tests.utils.bases.http_request", "tests.utils.bases.http_response", "tests.utils.bases.spider", diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 07ff4a74e..29b3e3c0f 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -126,5 +126,4 @@ class FTPDownloadHandler(BaseDownloadHandler): headers = {"local filename": protocol.filename or b"", "size": protocol.size} body = protocol.filename or protocol.body.read() respcls = responsetypes.from_args(url=request.url, body=body) - # hints for Headers-related types may need to be fixed to not use AnyStr - return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type] + return respcls(url=request.url, status=200, body=body, headers=headers) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 34d4ec6f2..b55ef6191 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast +from typing import TYPE_CHECKING, Any, TypeAlias, cast from w3lib.http import headers_dict_to_raw @@ -25,14 +25,20 @@ class Headers(CaselessDict): def __init__( self, - seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + seq: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, encoding: str = "utf-8", ): self.encoding: str = encoding super().__init__(seq) def update( # type: ignore[override] - self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] + self, + seq: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]], ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq iseq: dict[bytes, list[bytes]] = {} @@ -40,7 +46,7 @@ class Headers(CaselessDict): iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) super().update(iseq) - def normkey(self, key: AnyStr) -> bytes: # type: ignore[override] + def normkey(self, key: str | bytes) -> bytes: """Normalize key to bytes""" return self._tobytes(key.title()) @@ -67,19 +73,19 @@ class Headers(CaselessDict): return str(x).encode(self.encoding) raise TypeError(f"Unsupported value type: {type(x)}") - def __getitem__(self, key: AnyStr) -> bytes | None: + def __getitem__(self, key: str | bytes) -> bytes | None: try: return cast("list[bytes]", super().__getitem__(key))[-1] except IndexError: return None - def get(self, key: AnyStr, def_val: Any = None) -> bytes | None: + def get(self, key: str | bytes, def_val: Any = None) -> bytes | None: try: return cast("list[bytes]", super().get(key, def_val))[-1] except IndexError: return None - def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]: + def getlist(self, key: str | bytes, def_val: Any = None) -> list[bytes]: try: return cast("list[bytes]", super().__getitem__(key)) except KeyError: @@ -87,15 +93,15 @@ class Headers(CaselessDict): return self.normvalue(def_val) return [] - def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None: + def setlist(self, key: str | bytes, list_: Iterable[_RawValue]) -> None: self[key] = list_ def setlistdefault( - self, key: AnyStr, default_list: Iterable[_RawValue] = () + self, key: str | bytes, default_list: Iterable[_RawValue] = () ) -> Any: return self.setdefault(key, default_list) - def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None: + def appendlist(self, key: str | bytes, value: Iterable[_RawValue]) -> None: lst = self.getlist(key) lst.extend(self.normvalue(value)) self[key] = lst diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 73c2e7dd4..57517a65b 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -11,7 +11,6 @@ import inspect from typing import ( TYPE_CHECKING, Any, - AnyStr, Concatenate, NoReturn, TypeAlias, @@ -125,7 +124,10 @@ class Request(object_ref): url: str, callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, @@ -310,7 +312,11 @@ class Request(object_ref): @headers.setter def headers( - self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None + self, + value: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None, ) -> None: if isinstance(value, Headers): self._headers = value diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 09b1c8b32..f1db11488 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload +from typing import TYPE_CHECKING, Any, TypeVar, overload from urllib.parse import urljoin from scrapy.exceptions import NotSupported @@ -72,7 +72,10 @@ class Response(object_ref): self, url: str, status: int = 200, - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes = b"", flags: list[str] | None = None, request: Request | None = None, @@ -145,7 +148,11 @@ class Response(object_ref): @headers.setter def headers( - self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None + self, + value: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None, ) -> None: if isinstance(value, Headers): self._headers = value @@ -222,7 +229,10 @@ class Response(object_ref): url: str | Link, callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, @@ -272,7 +282,10 @@ class Response(object_ref): urls: Iterable[str | Link], callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6876e35e8..d01e23e47 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -9,7 +9,7 @@ from __future__ import annotations import json from contextlib import suppress -from typing import TYPE_CHECKING, Any, AnyStr, cast +from typing import TYPE_CHECKING, Any, cast from urllib.parse import urljoin import parsel @@ -170,7 +170,10 @@ class TextResponse(Response): url: str | Link | parsel.Selector, callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, @@ -223,7 +226,10 @@ class TextResponse(Response): urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None, callback: CallbackT | None = None, method: str = "GET", - headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + headers: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, body: bytes | str | None = None, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index e761a2474..9a945c61c 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -11,12 +11,12 @@ import warnings import weakref from collections import OrderedDict from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast +from typing import TYPE_CHECKING, Any, TypeVar, cast from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: - from collections.abc import Iterable, Sequence + from collections.abc import Container, Iterable # typing.Self requires Python 3.11 from typing_extensions import Self @@ -44,22 +44,25 @@ class CaselessDict(dict): # type: ignore[type-arg] def __init__( self, - seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + seq: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]] + | None = None, ): super().__init__() if seq: self.update(seq) - def __getitem__(self, key: AnyStr) -> Any: + def __getitem__(self, key: str | bytes) -> Any: return dict.__getitem__(self, self.normkey(key)) - def __setitem__(self, key: AnyStr, value: Any) -> None: + def __setitem__(self, key: str | bytes, value: Any) -> None: dict.__setitem__(self, self.normkey(key), self.normvalue(value)) - def __delitem__(self, key: AnyStr) -> None: + def __delitem__(self, key: str | bytes) -> None: dict.__delitem__(self, self.normkey(key)) - def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] + def __contains__(self, key: str | bytes) -> bool: # type: ignore[override] return dict.__contains__(self, self.normkey(key)) has_key = __contains__ @@ -69,7 +72,7 @@ class CaselessDict(dict): # type: ignore[type-arg] copy = __copy__ - def normkey(self, key: AnyStr) -> AnyStr: + def normkey(self, key: str | bytes) -> str | bytes: """Method to normalize dictionary key access""" return key.lower() @@ -77,23 +80,28 @@ class CaselessDict(dict): # type: ignore[type-arg] """Method to normalize values prior to be set""" return value - def get(self, key: AnyStr, def_val: Any = None) -> Any: + def get(self, key: str | bytes, def_val: Any = None) -> Any: return dict.get(self, self.normkey(key), self.normvalue(def_val)) - def setdefault(self, key: AnyStr, def_val: Any = None) -> Any: + def setdefault(self, key: str | bytes, def_val: Any = None) -> Any: return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # doesn't fully implement MutableMapping.update() - def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None: # type: ignore[override] + def update( # type: ignore[override] + self, + seq: Mapping[str, Any] + | Mapping[bytes, Any] + | Iterable[tuple[str | bytes, Any]], + ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super().update(iseq) @classmethod - def fromkeys(cls, keys: Iterable[AnyStr], value: Any = None) -> Self: # type: ignore[override] - return cls((k, value) for k in keys) # type: ignore[misc] + def fromkeys(cls, keys: Iterable[str | bytes], value: Any = None) -> Self: # type: ignore[override] + return cls((k, value) for k in keys) - def pop(self, key: AnyStr, *args: Any) -> Any: + def pop(self, key: str | bytes, *args: Any) -> Any: return dict.pop(self, self.normkey(key), *args) @@ -205,8 +213,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]): class SequenceExclude: """Object to test if an item is NOT within some sequence.""" - def __init__(self, seq: Sequence[Any]): - self.seq: Sequence[Any] = seq + def __init__(self, seq: Container[Any]): + self.seq: Container[Any] = seq def __contains__(self, item: Any) -> bool: return item not in self.seq diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index a5bb6fa24..4960dc27a 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -19,9 +19,19 @@ _T = TypeVar("_T") _P = ParamSpec("_P") +@overload +def deprecated(use_instead: Callable[_P, _T]) -> Callable[_P, _T]: ... + + +@overload def deprecated( - use_instead: Any = None, -) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: + use_instead: str | None = None, +) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: ... + + +def deprecated( + use_instead: Callable[_P, _T] | str | None = None, +) -> Callable[_P, _T] | Callable[[Callable[_P, _T]], Callable[_P, _T]]: """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.""" @@ -38,8 +48,9 @@ def deprecated( return wrapped if callable(use_instead): - deco = deco(use_instead) + func = use_instead use_instead = None + return deco(func) return deco diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index fc4e1c487..1d36a66fc 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -1,16 +1,18 @@ +from __future__ import annotations + from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from tests.utils.decorators import coroutine_test -class TestAsyncgenUtils: - @coroutine_test - async def test_as_async_generator(self): - ag = as_async_generator(range(42)) - results = [i async for i in ag] - assert results == list(range(42)) +@coroutine_test +async def test_as_async_generator(): + ag = as_async_generator(range(42)) + results = [i async for i in ag] + assert results == list(range(42)) - @coroutine_test - async def test_collect_asyncgen(self): - ag = as_async_generator(range(42)) - results = await collect_asyncgen(ag) - assert results == list(range(42)) + +@coroutine_test +async def test_collect_asyncgen(): + ag = as_async_generator(range(42)) + results = await collect_asyncgen(ag) + assert results == list(range(42)) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 7528dc51a..9b7eb22fa 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -20,11 +20,10 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator -class TestAsyncio: - @coroutine_test - async def test_is_asyncio_available(self, reactor_pytest: str) -> None: - # the result should depend only on the pytest --reactor argument - assert is_asyncio_available() == (reactor_pytest != "default") +@coroutine_test +async def test_is_asyncio_available(reactor_pytest: str) -> None: + # the result should depend only on the pytest --reactor argument + assert is_asyncio_available() == (reactor_pytest != "default") @pytest.mark.only_asyncio diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index fce9fc984..6b30744bb 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import warnings from typing import Any diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index f43d20e69..af203ca61 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,7 +1,9 @@ +from __future__ import annotations + import copy from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, MutableMapping -from typing import Any +from typing import Any, Generic, TypeVar import pytest @@ -16,11 +18,13 @@ from scrapy.utils.datatypes import ( ) from scrapy.utils.python import garbage_collect +_DictT = TypeVar("_DictT", bound="CaselessDict | CaseInsensitiveDict") -class TestCaseInsensitiveDictBase(ABC): + +class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]): @property @abstractmethod - def dict_class(self) -> type[MutableMapping[str, Any]]: + def dict_class(self) -> type[_DictT]: raise NotImplementedError def test_init_dict(self): @@ -36,17 +40,17 @@ class TestCaseInsensitiveDictBase(ABC): assert d["black"] == 3 def test_init_mapping(self): - class MyMapping(Mapping): - def __init__(self, **kwargs): + class MyMapping(Mapping[str, int]): + def __init__(self, **kwargs: int) -> None: self._d = kwargs - def __getitem__(self, key): + def __getitem__(self, key: str) -> int: return self._d[key] - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._d) - def __len__(self): + def __len__(self) -> int: return len(self._d) seq = MyMapping(red=1, black=3) @@ -55,23 +59,23 @@ class TestCaseInsensitiveDictBase(ABC): assert d["black"] == 3 def test_init_mutable_mapping(self): - class MyMutableMapping(MutableMapping): - def __init__(self, **kwargs): + class MyMutableMapping(MutableMapping[str, int]): + def __init__(self, **kwargs: int) -> None: self._d = kwargs - def __getitem__(self, key): + def __getitem__(self, key: str) -> int: return self._d[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: int) -> None: self._d[key] = value - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: del self._d[key] - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._d) - def __len__(self): + def __len__(self) -> int: return len(self._d) seq = MyMutableMapping(red=1, black=3) @@ -149,7 +153,7 @@ class TestCaseInsensitiveDictBase(ABC): d.pop("A") def test_normkey(self): - class MyDict(self.dict_class): + class MyDict(self.dict_class): # type: ignore[misc,name-defined] def _normkey(self, key): return key.title() @@ -160,7 +164,7 @@ class TestCaseInsensitiveDictBase(ABC): assert list(d.keys()) == ["Key-One"] def test_normvalue(self): - class MyDict(self.dict_class): + class MyDict(self.dict_class): # type: ignore[misc,name-defined] def _normvalue(self, value): if value is not None: return value + 1 @@ -214,8 +218,8 @@ class TestCaseInsensitiveDictBase(ABC): assert dict(h1) == {"header1": "value1", "header2": "value2"} -class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): - dict_class = CaseInsensitiveDict # type: ignore[assignment] +class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase[CaseInsensitiveDict]): + dict_class = CaseInsensitiveDict def test_repr(self): d1 = self.dict_class({"foo": "bar"}) @@ -230,7 +234,7 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): assert list(iterkeys) == ["AsDf", "FoO"] def test_copy_keeps_values(self): - class MyDict(self.dict_class): + class MyDict(self.dict_class): # type: ignore[misc,name-defined] def _normvalue(self, value): return value + 1 @@ -253,7 +257,7 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestCaselessDict(TestCaseInsensitiveDictBase): +class TestCaselessDict(TestCaseInsensitiveDictBase[CaselessDict]): dict_class = CaselessDict def test_deprecation_message(self): @@ -319,7 +323,7 @@ class TestSequenceExclude: class TestLocalCache: def test_cache_with_limit(self): - cache = LocalCache(limit=2) + cache: LocalCache[str, int] = LocalCache(limit=2) cache["a"] = 1 cache["b"] = 2 cache["c"] = 3 @@ -332,7 +336,7 @@ class TestLocalCache: def test_cache_without_limit(self): maximum = 10**4 - cache = LocalCache() + cache: LocalCache[str, int] = LocalCache() for x in range(maximum): cache[str(x)] = x assert len(cache) == maximum @@ -341,7 +345,7 @@ class TestLocalCache: assert cache[str(x)] == x def test_cache_with_zero_limit(self): - cache = LocalCache(limit=0) + cache: LocalCache[str, int] = LocalCache(limit=0) cache["a"] = 1 cache["b"] = 2 cache["c"] = 3 @@ -353,7 +357,9 @@ class TestLocalCache: class TestLocalWeakReferencedCache: def test_cache_with_limit(self): - cache = LocalWeakReferencedCache(limit=2) + cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache( + limit=2 + ) r1 = Request("https://example.org") r2 = Request("https://example.com") r3 = Request("https://example.net") @@ -375,7 +381,7 @@ class TestLocalWeakReferencedCache: assert len(cache) == 1 def test_cache_non_weak_referenceable_objects(self): - cache = LocalWeakReferencedCache() + cache: LocalWeakReferencedCache[Any, int] = LocalWeakReferencedCache() k1 = None k2 = 1 k3 = [1, 2, 3] @@ -389,7 +395,7 @@ class TestLocalWeakReferencedCache: def test_cache_without_limit(self): maximum = 10**4 - cache = LocalWeakReferencedCache() + cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache() refs = [] for x in range(maximum): refs.append(Request(f"https://example.org/{x}")) diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.py index 9743e1a50..807294a57 100644 --- a/tests/test_utils_decorators.py +++ b/tests/test_utils_decorators.py @@ -1,6 +1,7 @@ from __future__ import annotations import warnings +from typing import TYPE_CHECKING import pytest from twisted.internet.defer import Deferred @@ -10,11 +11,14 @@ from scrapy.utils.decorators import _warn_spider_arg, deprecated, inthread from scrapy.utils.defer import maybe_deferred_to_future from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + class TestDeprecated: def test_warns_and_still_calls(self): @deprecated() - def add(a, b): + def add(a: int, b: int) -> int: return a + b with pytest.warns( @@ -26,7 +30,7 @@ class TestDeprecated: def test_use_instead_in_message(self): @deprecated(use_instead="other_function") - def old(): + def old() -> None: return None with pytest.warns( @@ -37,7 +41,7 @@ class TestDeprecated: def test_applied_without_parentheses(self): @deprecated - def square(x): + def square(x: int) -> int: return x * x with pytest.warns( @@ -65,7 +69,7 @@ class TestInthread: class TestWarnSpiderArg: def test_sync_warns_with_spider_arg(self): @_warn_spider_arg - def parse(response, spider=None): + def parse(response: str, spider: str | None = None) -> str: return response with pytest.warns( @@ -75,7 +79,7 @@ class TestWarnSpiderArg: def test_sync_no_warning_without_spider_arg(self): @_warn_spider_arg - def parse(response, spider=None): + def parse(response: str, spider: str | None = None) -> str: return response with warnings.catch_warnings(): @@ -85,7 +89,7 @@ class TestWarnSpiderArg: @coroutine_test async def test_async_warns_with_spider_arg(self): @_warn_spider_arg - async def parse(response, spider=None): + async def parse(response: str, spider: str | None = None) -> str: return response with pytest.warns( @@ -96,7 +100,9 @@ class TestWarnSpiderArg: @coroutine_test async def test_asyncgen_warns_with_spider_arg(self): @_warn_spider_arg - async def parse(response, spider=None): + async def parse( + response: str, spider: str | None = None + ) -> AsyncGenerator[str]: yield response with pytest.warns( diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 978c24f5a..175a4fe03 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -24,6 +24,8 @@ from tests.utils.decorators import coroutine_test, inline_callbacks_test if TYPE_CHECKING: from collections.abc import AsyncGenerator, Awaitable, Callable, Generator + from twisted.python.failure import Failure + @pytest.mark.requires_reactor # mustbe_deferred() requires a reactor @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") @@ -70,7 +72,7 @@ class TestIterErrback: def itergood() -> Generator[int, None, None]: yield from range(10) - errors = [] + errors: list[Failure] = [] out = list(iter_errback(itergood(), errors.append)) assert out == list(range(10)) assert not errors @@ -82,7 +84,7 @@ class TestIterErrback: 1 / 0 yield x - errors = [] + errors: list[Failure] = [] out = list(iter_errback(iterbad(), errors.append)) assert out == [0, 1, 2, 3, 4] assert len(errors) == 1 @@ -96,7 +98,7 @@ class TestAiterErrback: for x in range(10): yield x - errors = [] + errors: list[Failure] = [] out = await collect_asyncgen(aiter_errback(itergood(), errors.append)) assert out == list(range(10)) assert not errors @@ -109,7 +111,7 @@ class TestAiterErrback: 1 / 0 yield x - errors = [] + errors: list[Failure] = [] out = await collect_asyncgen(aiter_errback(iterbad(), errors.append)) assert out == [0, 1, 2, 3, 4] assert len(errors) == 1 @@ -202,7 +204,7 @@ class TestParallelAsync: for length in [20, 50, 100]: parallel_count = [0] max_parallel_count = [0] - results = [] + results: list[int] = [] ait = self.get_async_iterable(length) dl = parallel_async( ait, @@ -222,7 +224,7 @@ class TestParallelAsync: for length in [20, 50, 100]: parallel_count = [0] max_parallel_count = [0] - results = [] + results: list[int] = [] ait = self.get_async_iterable_with_delays(length) dl = parallel_async( ait, @@ -240,7 +242,7 @@ class TestParallelAsync: class TestDeferredFromCoro: def test_deferred(self): - d = Deferred() + d: Deferred[None] = Deferred() result = deferred_from_coro(d) assert isinstance(result, Deferred) assert result is d @@ -274,7 +276,7 @@ class TestDeferredFromCoro: @pytest.mark.only_asyncio @inline_callbacks_test def test_future(self): - future = Future() + future: Future[int] = Future() result = deferred_from_coro(future) assert isinstance(result, Deferred) future.set_result(42) @@ -324,7 +326,7 @@ class TestDeferredFFromCoroF: class TestDeferredToFuture: @coroutine_test async def test_deferred(self): - d = Deferred() + d: Deferred[int] = Deferred() result = deferred_to_future(d) assert isinstance(result, Future) d.callback(42) @@ -359,7 +361,7 @@ class TestDeferredToFuture: class TestMaybeDeferredToFutureAsyncio: @coroutine_test async def test_deferred(self): - d = Deferred() + d: Deferred[int] = Deferred() result = maybe_deferred_to_future(d) assert isinstance(result, Future) d.callback(42) @@ -394,7 +396,7 @@ class TestMaybeDeferredToFutureAsyncio: class TestMaybeDeferredToFutureNotAsyncio: @coroutine_test async def test_deferred(self): - d = Deferred() + d: Deferred[int] = Deferred() result = maybe_deferred_to_future(d) assert isinstance(result, Deferred) assert result is d diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 0706fec99..4c8585916 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import inspect import warnings from unittest import mock @@ -38,7 +40,7 @@ class TestWarnWhenSubclassed: ) with pytest.warns(MyWarning, match=msg) as w: - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass assert w[0].lineno == inspect.getsourcelines(UserClass)[1] @@ -57,7 +59,7 @@ class TestWarnWhenSubclassed: match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass", ): - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass with pytest.warns( @@ -76,7 +78,7 @@ class TestWarnWhenSubclassed: match="UserClass inherits from deprecated class", ): - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass with warnings.catch_warnings(): @@ -95,16 +97,16 @@ class TestWarnWhenSubclassed: match="UserClass inherits from deprecated class", ): - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass with warnings.catch_warnings(): warnings.simplefilter("error", MyWarning) - class FooClass(Deprecated): + class FooClass(Deprecated): # type: ignore[misc, valid-type] pass - class BarClass(Deprecated): + class BarClass(Deprecated): # type: ignore[misc, valid-type] pass def test_warning_on_instance(self): @@ -112,22 +114,20 @@ class TestWarnWhenSubclassed: "Deprecated", NewName, warn_category=MyWarning ) - with pytest.warns(MyWarning) as w: - _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) - - w = [x for x in w if x.category is MyWarning] + with pytest.warns( + MyWarning, + match=r"tests\.test_utils_deprecate\.Deprecated is deprecated, " + r"instantiate tests\.test_utils_deprecate\.NewName instead\.", + ) as w: + _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) # type: ignore[arg-type] assert len(w) == 1 - assert ( - str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, " - "instantiate tests.test_utils_deprecate.NewName instead." - ) assert w[0].lineno == lineno # ignore subclassing warnings with warnings.catch_warnings(): warnings.simplefilter("ignore", MyWarning) - class UserClass(Deprecated): + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass with warnings.catch_warnings(): @@ -141,7 +141,7 @@ class TestWarnWhenSubclassed: match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName", ): - class UserClass2(Deprecated): + class UserClass2(Deprecated): # type: ignore[misc, valid-type] pass def test_issubclass(self): @@ -155,10 +155,10 @@ class TestWarnWhenSubclassed: class UpdatedUserClass1a(NewName): pass - class OutdatedUserClass1(DeprecatedName): + class OutdatedUserClass1(DeprecatedName): # type: ignore[misc, valid-type] pass - class OutdatedUserClass1a(DeprecatedName): + class OutdatedUserClass1a(DeprecatedName): # type: ignore[misc, valid-type] pass class UnrelatedClass: @@ -174,7 +174,7 @@ class TestWarnWhenSubclassed: assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1) with pytest.raises(TypeError): - issubclass(object(), DeprecatedName) + issubclass(object(), DeprecatedName) # type: ignore[arg-type] def test_isinstance(self): with warnings.catch_warnings(): @@ -187,10 +187,10 @@ class TestWarnWhenSubclassed: class UpdatedUserClass2a(NewName): pass - class OutdatedUserClass2(DeprecatedName): + class OutdatedUserClass2(DeprecatedName): # type: ignore[misc, valid-type] pass - class OutdatedUserClass2a(DeprecatedName): + class OutdatedUserClass2a(DeprecatedName): # type: ignore[misc, valid-type] pass class UnrelatedClass: @@ -211,7 +211,7 @@ class TestWarnWhenSubclassed: warnings.simplefilter("ignore", ScrapyDeprecationWarning) Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"}) - assert Deprecated.foo == "bar" + assert Deprecated.foo == "bar" # type: ignore[attr-defined] def test_deprecate_a_class_with_custom_metaclass(self): Meta1 = type("Meta1", (type,), {}) @@ -242,7 +242,7 @@ class TestWarnWhenSubclassed: match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar", ): - class UserClass(AlsoDeprecated): + class UserClass(AlsoDeprecated): # type: ignore[misc, valid-type] pass def test_inspect_stack(self): diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index 9f9e24957..a87816c20 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -31,13 +31,13 @@ plain_string = "{'a': 1}" @mock.patch("sys.platform", "linux") @mock.patch("sys.stdout.isatty") -def test_pformat(isatty): +def test_pformat(isatty: mock.Mock) -> None: isatty.return_value = True assert pformat(value) in colorized_strings @mock.patch("sys.stdout.isatty") -def test_pformat_dont_colorize(isatty): +def test_pformat_dont_colorize(isatty: mock.Mock) -> None: isatty.return_value = True assert pformat(value, colorize=False) == plain_string @@ -49,7 +49,7 @@ def test_pformat_not_tty(): @mock.patch("sys.platform", "win32") @mock.patch("platform.version") @mock.patch("sys.stdout.isatty") -def test_pformat_old_windows(isatty, version): +def test_pformat_old_windows(isatty: mock.Mock, version: mock.Mock) -> None: isatty.return_value = True version.return_value = "10.0.14392" assert pformat(value) in colorized_strings @@ -59,7 +59,9 @@ def test_pformat_old_windows(isatty, version): @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") @mock.patch("platform.version") @mock.patch("sys.stdout.isatty") -def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing): +def test_pformat_windows_no_terminal_processing( + isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock +) -> None: isatty.return_value = True version.return_value = "10.0.14393" terminal_processing.return_value = False @@ -70,7 +72,9 @@ def test_pformat_windows_no_terminal_processing(isatty, version, terminal_proces @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") @mock.patch("platform.version") @mock.patch("sys.stdout.isatty") -def test_pformat_windows(isatty, version, terminal_processing): +def test_pformat_windows( + isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock +) -> None: isatty.return_value = True version.return_value = "10.0.14393" terminal_processing.return_value = True @@ -79,7 +83,7 @@ def test_pformat_windows(isatty, version, terminal_processing): @mock.patch("sys.platform", "linux") @mock.patch("sys.stdout.isatty") -def test_pformat_no_pygments(isatty): +def test_pformat_no_pygments(isatty: mock.Mock) -> None: isatty.return_value = True real_import = builtins.__import__ diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 06fdf9cba..75f8be6f6 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from gzip import BadGzipFile from pathlib import Path diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 0eb330461..610c463ec 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from urllib.parse import urlparse from scrapy.http import Request diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index ab6965ed5..0775ec608 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -20,150 +20,157 @@ from scrapy.utils.misc import ( ) -class TestUtilsMisc: - def test_load_object_class(self): - obj = load_object(Field) - assert obj is Field - obj = load_object("scrapy.item.Field") - assert obj is Field +def test_load_object_class() -> None: + obj = load_object(Field) + assert obj is Field + obj = load_object("scrapy.item.Field") + assert obj is Field - def test_load_object_function(self): - obj = load_object(load_object) - assert obj is load_object - obj = load_object("scrapy.utils.misc.load_object") - assert obj is load_object - def test_load_object_exceptions(self): - with pytest.raises(ImportError): - load_object("nomodule999.mod.function") - with pytest.raises(NameError): - load_object("scrapy.utils.misc.load_object999") - with pytest.raises(TypeError): - load_object({}) # type: ignore[arg-type] +def test_load_object_function() -> None: + obj = load_object(load_object) + assert obj is load_object + obj = load_object("scrapy.utils.misc.load_object") + assert obj is load_object - def test_walk_modules(self): - mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules") + +def test_load_object_exceptions() -> None: + with pytest.raises(ImportError): + load_object("nomodule999.mod.function") + with pytest.raises(NameError): + load_object("scrapy.utils.misc.load_object999") + with pytest.raises(TypeError): + load_object({}) # type: ignore[arg-type] + + +def test_walk_modules() -> None: + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules") + expected = [ + "tests.test_utils_misc.test_walk_modules", + "tests.test_utils_misc.test_walk_modules.mod", + "tests.test_utils_misc.test_walk_modules.mod.mod0", + "tests.test_utils_misc.test_walk_modules.mod1", + ] + assert {m.__name__ for m in mods} == set(expected) + + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod") + expected = [ + "tests.test_utils_misc.test_walk_modules.mod", + "tests.test_utils_misc.test_walk_modules.mod.mod0", + ] + assert {m.__name__ for m in mods} == set(expected) + + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1") + expected = [ + "tests.test_utils_misc.test_walk_modules.mod1", + ] + assert {m.__name__ for m in mods} == set(expected) + + with pytest.raises(ImportError): + for _ in walk_modules_iter("nomodule999"): + pass + with ( + pytest.raises(ImportError), + pytest.warns( + ScrapyDeprecationWarning, + match="The scrapy.utils.misc.walk_modules function is deprecated and will be " + "removed in a future version of Scrapy. " + "Use scrapy.utils.misc.walk_modules_iter instead.", + ), + ): + walk_modules("nomodule999") + + +def test_walk_modules_egg() -> None: + egg = str(Path(__file__).parent / "test.egg") + sys.path.append(egg) + try: + mods = walk_modules_iter("testegg") expected = [ - "tests.test_utils_misc.test_walk_modules", - "tests.test_utils_misc.test_walk_modules.mod", - "tests.test_utils_misc.test_walk_modules.mod.mod0", - "tests.test_utils_misc.test_walk_modules.mod1", + "testegg.spiders", + "testegg.spiders.a", + "testegg.spiders.b", + "testegg", ] assert {m.__name__ for m in mods} == set(expected) + finally: + sys.path.remove(egg) - mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod") - expected = [ - "tests.test_utils_misc.test_walk_modules.mod", - "tests.test_utils_misc.test_walk_modules.mod.mod0", - ] - assert {m.__name__ for m in mods} == set(expected) - mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1") - expected = [ - "tests.test_utils_misc.test_walk_modules.mod1", - ] - assert {m.__name__ for m in mods} == set(expected) +def test_arg_to_iter() -> None: + class TestItem(Item): + name = Field() - with pytest.raises(ImportError): - for _ in walk_modules_iter("nomodule999"): - pass - with ( - pytest.raises(ImportError), - pytest.warns( - ScrapyDeprecationWarning, - match="The scrapy.utils.misc.walk_modules function is deprecated and will be " - "removed in a future version of Scrapy. " - "Use scrapy.utils.misc.walk_modules_iter instead.", - ), - ): - walk_modules("nomodule999") + assert hasattr(arg_to_iter(None), "__iter__") + assert hasattr(arg_to_iter(100), "__iter__") + assert hasattr(arg_to_iter("lala"), "__iter__") + assert hasattr(arg_to_iter([1, 2, 3]), "__iter__") + assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__") - def test_walk_modules_egg(self): - egg = str(Path(__file__).parent / "test.egg") - sys.path.append(egg) - try: - mods = walk_modules_iter("testegg") - expected = [ - "testegg.spiders", - "testegg.spiders.a", - "testegg.spiders.b", - "testegg", - ] - assert {m.__name__ for m in mods} == set(expected) - finally: - sys.path.remove(egg) + assert not list(arg_to_iter(None)) + assert list(arg_to_iter("lala")) == ["lala"] + assert list(arg_to_iter(100)) == [100] + assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"] + assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3] + assert list(arg_to_iter({"a": 1})) == [{"a": 1}] + assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")] - def test_arg_to_iter(self): - class TestItem(Item): - name = Field() - assert hasattr(arg_to_iter(None), "__iter__") - assert hasattr(arg_to_iter(100), "__iter__") - assert hasattr(arg_to_iter("lala"), "__iter__") - assert hasattr(arg_to_iter([1, 2, 3]), "__iter__") - assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__") +def test_build_from_crawler() -> None: + crawler = mock.MagicMock(spec_set=["settings"]) + args = (True, 100.0) + kwargs = {"key": "val"} - assert not list(arg_to_iter(None)) - assert list(arg_to_iter("lala")) == ["lala"] - assert list(arg_to_iter(100)) == [100] - assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"] - assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3] - assert list(arg_to_iter({"a": 1})) == [{"a": 1}] - assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")] + def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None: + build_from_crawler(mock, crawler, *args, **kwargs) + if hasattr(mock, "from_crawler"): + mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) + assert mock.call_count == 0 + else: + mock.assert_called_once_with(*args, **kwargs) - def test_build_from_crawler(self): - crawler = mock.MagicMock(spec_set=["settings"]) - args = (True, 100.0) - kwargs = {"key": "val"} + # Check usage of correct constructor using 2 mocks: + # 1. with no alternative constructors + # 2. with from_crawler() constructor + spec_sets = ( + ["__qualname__"], + ["__qualname__", "from_crawler"], + ) + for specs in spec_sets: + m = mock.MagicMock(spec_set=specs) + _test_with_crawler(m, crawler) + m.reset_mock() - def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None: - build_from_crawler(mock, crawler, *args, **kwargs) - if hasattr(mock, "from_crawler"): - mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) - assert mock.call_count == 0 - else: - mock.assert_called_once_with(*args, **kwargs) + # Check adoption of crawler + m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"]) + m.from_crawler.return_value = None + with pytest.raises(TypeError): + build_from_crawler(m, crawler, *args, **kwargs) - # Check usage of correct constructor using 2 mocks: - # 1. with no alternative constructors - # 2. with from_crawler() constructor - spec_sets = ( - ["__qualname__"], - ["__qualname__", "from_crawler"], - ) - for specs in spec_sets: - m = mock.MagicMock(spec_set=specs) - _test_with_crawler(m, crawler) - m.reset_mock() - # Check adoption of crawler - m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"]) - m.from_crawler.return_value = None - with pytest.raises(TypeError): - build_from_crawler(m, crawler, *args, **kwargs) +def test_set_environ() -> None: + assert os.environ.get("some_test_environ") is None + with set_environ(some_test_environ="test_value"): + assert os.environ.get("some_test_environ") == "test_value" + assert os.environ.get("some_test_environ") is None - def test_set_environ(self): - assert os.environ.get("some_test_environ") is None - with set_environ(some_test_environ="test_value"): - assert os.environ.get("some_test_environ") == "test_value" - assert os.environ.get("some_test_environ") is None + os.environ["some_test_environ"] = "test" + assert os.environ.get("some_test_environ") == "test" + with set_environ(some_test_environ="test_value"): + assert os.environ.get("some_test_environ") == "test_value" + assert os.environ.get("some_test_environ") == "test" - os.environ["some_test_environ"] = "test" - assert os.environ.get("some_test_environ") == "test" - with set_environ(some_test_environ="test_value"): - assert os.environ.get("some_test_environ") == "test_value" - assert os.environ.get("some_test_environ") == "test" - def test_rel_has_nofollow(self): - assert rel_has_nofollow("ugc nofollow") is True - assert rel_has_nofollow("ugc,nofollow") is True - assert rel_has_nofollow("ugc") is False - assert rel_has_nofollow("nofollow") is True - assert rel_has_nofollow("nofollowfoo") is False - assert rel_has_nofollow("foonofollow") is False - assert rel_has_nofollow("ugc, , nofollow") is True - # rel attribute values are ASCII case-insensitive per the HTML spec - assert rel_has_nofollow("NoFollow") is True - assert rel_has_nofollow("NOFOLLOW") is True - assert rel_has_nofollow("UGC NoFollow") is True - assert rel_has_nofollow("ugc,NoFollow") is True +def test_rel_has_nofollow() -> None: + assert rel_has_nofollow("ugc nofollow") is True + assert rel_has_nofollow("ugc,nofollow") is True + assert rel_has_nofollow("ugc") is False + assert rel_has_nofollow("nofollow") is True + assert rel_has_nofollow("nofollowfoo") is False + assert rel_has_nofollow("foonofollow") is False + assert rel_has_nofollow("ugc, , nofollow") is True + # rel attribute values are ASCII case-insensitive per the HTML spec + assert rel_has_nofollow("NoFollow") is True + assert rel_has_nofollow("NOFOLLOW") is True + assert rel_has_nofollow("UGC NoFollow") is True + assert rel_has_nofollow("ugc,NoFollow") is True diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 1acc3aac2..7343a135a 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,5 +1,8 @@ +from __future__ import annotations + import warnings from functools import partial +from typing import TYPE_CHECKING, Any from unittest import mock import pytest @@ -9,6 +12,11 @@ from scrapy.utils.misc import ( warn_on_generator_with_return_value, ) +if TYPE_CHECKING: + from collections.abc import Generator + + from scrapy import Spider + def _indentation_error(*args, **kwargs): raise IndentationError @@ -35,244 +43,239 @@ https://example.org yield url -def generator_that_returns_stuff(): +def generator_that_returns_stuff() -> Generator[int, None, int]: yield 1 yield 2 return 3 -class TestUtilsMisc: - @pytest.fixture - def mock_spider(self): - class MockSettings: - def __init__(self, settings_dict=None): - self.settings_dict = settings_dict or { - "WARN_ON_GENERATOR_RETURN_VALUE": True - } +@pytest.fixture +def mock_spider() -> Spider: + class MockSettings: + def __init__(self, settings_dict: dict[str, Any] | None = None): + self.settings_dict = settings_dict or { + "WARN_ON_GENERATOR_RETURN_VALUE": True + } - def getbool(self, name, default=False): - return self.settings_dict.get(name, default) + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) - class MockSpider: - def __init__(self): - self.settings = MockSettings() + class MockSpider: + def __init__(self) -> None: + self.settings = MockSettings() - return MockSpider() + return MockSpider() # type: ignore[return-value] - def test_generators_return_something(self, mock_spider): - def f1(): - yield 1 - return 2 - def g1(): - yield 1 - return "asdf" +def test_generators_return_something(mock_spider): + def f1(): + yield 1 + return 2 - def h1(): - yield 1 + def g1(): + yield 1 + return "asdf" - def helper(): - return 0 + def h1(): + yield 1 - yield helper() - return 2 + def helper() -> int: + return 0 - def i1(): - """ - docstring - """ - url = """ -https://example.org + yield helper() + return 2 + + def i1(): """ - yield url - return 1 - - assert is_generator_with_return_value(top_level_return_something) - assert is_generator_with_return_value(f1) - assert is_generator_with_return_value(g1) - assert is_generator_with_return_value(h1) - assert is_generator_with_return_value(i1) - - with pytest.warns( - UserWarning, - match='The "MockSpider.top_level_return_something" method is a generator', - ): - warn_on_generator_with_return_value(mock_spider, top_level_return_something) - with pytest.warns( - UserWarning, match='The "MockSpider.f1" method is a generator' - ): - warn_on_generator_with_return_value(mock_spider, f1) - with pytest.warns( - UserWarning, match='The "MockSpider.g1" method is a generator' - ): - warn_on_generator_with_return_value(mock_spider, g1) - with pytest.warns( - UserWarning, match='The "MockSpider.h1" method is a generator' - ): - warn_on_generator_with_return_value(mock_spider, h1) - with pytest.warns( - UserWarning, match='The "MockSpider.i1" method is a generator' - ): - warn_on_generator_with_return_value(mock_spider, i1) - - def test_generators_return_none(self, mock_spider): - def f2(): - yield 1 - - def g2(): - yield 1 - - def h2(): - yield 1 - - def i2(): - yield 1 - yield from generator_that_returns_stuff() - - def j2(): - yield 1 - - def helper(): - return 0 - - yield helper() - - def k2(): - """ - docstring - """ - url = """ -https://example.org + docstring """ - yield url - - def l2(): - return - - assert not is_generator_with_return_value(top_level_return_none) - assert not is_generator_with_return_value(f2) - assert not is_generator_with_return_value(g2) - assert not is_generator_with_return_value(h2) - assert not is_generator_with_return_value(i2) - assert not is_generator_with_return_value(j2) # not recursive - assert not is_generator_with_return_value(k2) # not recursive - assert not is_generator_with_return_value(l2) - - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - warn_on_generator_with_return_value(mock_spider, top_level_return_none) - warn_on_generator_with_return_value(mock_spider, f2) - warn_on_generator_with_return_value(mock_spider, g2) - warn_on_generator_with_return_value(mock_spider, h2) - warn_on_generator_with_return_value(mock_spider, i2) - warn_on_generator_with_return_value(mock_spider, j2) - warn_on_generator_with_return_value(mock_spider, k2) - warn_on_generator_with_return_value(mock_spider, l2) - - def test_generators_return_none_with_decorator(self, mock_spider): - def decorator(func): - def inner_func(): - func() - - return inner_func - - @decorator - def f3(): - yield 1 - - @decorator - def g3(): - yield 1 - - @decorator - def h3(): - yield 1 - - @decorator - def i3(): - yield 1 - yield from generator_that_returns_stuff() - - @decorator - def j3(): - yield 1 - - def helper(): - return 0 - - yield helper() - - @decorator - def k3(): - """ - docstring - """ - url = """ + url = """ https://example.org + """ + yield url + return 1 + + assert is_generator_with_return_value(top_level_return_something) + assert is_generator_with_return_value(f1) + assert is_generator_with_return_value(g1) + assert is_generator_with_return_value(h1) + assert is_generator_with_return_value(i1) + + with pytest.warns( + UserWarning, + match='The "MockSpider.top_level_return_something" method is a generator', + ): + warn_on_generator_with_return_value(mock_spider, top_level_return_something) + with pytest.warns(UserWarning, match='The "MockSpider.f1" method is a generator'): + warn_on_generator_with_return_value(mock_spider, f1) + with pytest.warns(UserWarning, match='The "MockSpider.g1" method is a generator'): + warn_on_generator_with_return_value(mock_spider, g1) + with pytest.warns(UserWarning, match='The "MockSpider.h1" method is a generator'): + warn_on_generator_with_return_value(mock_spider, h1) + with pytest.warns(UserWarning, match='The "MockSpider.i1" method is a generator'): + warn_on_generator_with_return_value(mock_spider, i1) + + +def test_generators_return_none(mock_spider): + def f2(): + yield 1 + + def g2(): + yield 1 + + def h2(): + yield 1 + + def i2(): + yield 1 + yield from generator_that_returns_stuff() + + def j2(): + yield 1 + + def helper() -> int: + return 0 + + yield helper() + + def k2(): """ - yield url + docstring + """ + url = """ +https://example.org + """ + yield url - @decorator - def l3(): - return + def l2(): + return - assert not is_generator_with_return_value(top_level_return_none) - assert not is_generator_with_return_value(f3) - assert not is_generator_with_return_value(g3) - assert not is_generator_with_return_value(h3) - assert not is_generator_with_return_value(i3) - assert not is_generator_with_return_value(j3) # not recursive - assert not is_generator_with_return_value(k3) # not recursive - assert not is_generator_with_return_value(l3) + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f2) + assert not is_generator_with_return_value(g2) + assert not is_generator_with_return_value(h2) + assert not is_generator_with_return_value(i2) + assert not is_generator_with_return_value(j2) # not recursive + assert not is_generator_with_return_value(k2) # not recursive + assert not is_generator_with_return_value(l2) - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - warn_on_generator_with_return_value(mock_spider, top_level_return_none) - warn_on_generator_with_return_value(mock_spider, f3) - warn_on_generator_with_return_value(mock_spider, g3) - warn_on_generator_with_return_value(mock_spider, h3) - warn_on_generator_with_return_value(mock_spider, i3) - warn_on_generator_with_return_value(mock_spider, j3) - warn_on_generator_with_return_value(mock_spider, k3) - warn_on_generator_with_return_value(mock_spider, l3) + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, f2) + warn_on_generator_with_return_value(mock_spider, g2) + warn_on_generator_with_return_value(mock_spider, h2) + warn_on_generator_with_return_value(mock_spider, i2) + warn_on_generator_with_return_value(mock_spider, j2) + warn_on_generator_with_return_value(mock_spider, k2) + warn_on_generator_with_return_value(mock_spider, l2) - @mock.patch( - "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error - ) - def test_indentation_error(self, mock_spider): - with pytest.warns(UserWarning, match="Unable to determine"): - warn_on_generator_with_return_value(mock_spider, top_level_return_none) - def test_partial(self): - def cb(arg1, arg2): - yield {} +def test_generators_return_none_with_decorator(mock_spider): + def decorator(func): + def inner_func(): + func() - partial_cb = partial(cb, arg1=42) - assert not is_generator_with_return_value(partial_cb) + return inner_func - def test_warn_on_generator_with_return_value_settings_disabled(self): - class MockSettings: - def __init__(self, settings_dict=None): - self.settings_dict = settings_dict or {} + @decorator + def f3(): + yield 1 - def getbool(self, name, default=False): - return self.settings_dict.get(name, default) + @decorator + def g3(): + yield 1 - class MockSpider: - def __init__(self): - self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False}) + @decorator + def h3(): + yield 1 - spider = MockSpider() + @decorator + def i3(): + yield 1 + yield from generator_that_returns_stuff() - def gen_with_return(): - yield 1 - return "value" + @decorator + def j3(): + yield 1 - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - warn_on_generator_with_return_value(spider, gen_with_return) + def helper() -> int: + return 0 - spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True + yield helper() - with pytest.warns(UserWarning, match="is a generator"): - warn_on_generator_with_return_value(spider, gen_with_return) + @decorator + def k3(): + """ + docstring + """ + url = """ +https://example.org + """ + yield url + + @decorator + def l3(): + return + + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f3) + assert not is_generator_with_return_value(g3) + assert not is_generator_with_return_value(h3) + assert not is_generator_with_return_value(i3) + assert not is_generator_with_return_value(j3) # not recursive + assert not is_generator_with_return_value(k3) # not recursive + assert not is_generator_with_return_value(l3) + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, f3) + warn_on_generator_with_return_value(mock_spider, g3) + warn_on_generator_with_return_value(mock_spider, h3) + warn_on_generator_with_return_value(mock_spider, i3) + warn_on_generator_with_return_value(mock_spider, j3) + warn_on_generator_with_return_value(mock_spider, k3) + warn_on_generator_with_return_value(mock_spider, l3) + + +@mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error) +def test_indentation_error(mock_spider): + with pytest.warns(UserWarning, match="Unable to determine"): + warn_on_generator_with_return_value(mock_spider, top_level_return_none) + + +def test_partial() -> None: + def cb(arg1, arg2): + yield {} + + partial_cb = partial(cb, arg1=42) + assert not is_generator_with_return_value(partial_cb) + + +def test_warn_on_generator_with_return_value_settings_disabled() -> None: + class MockSettings: + def __init__(self, settings_dict: dict[str, Any] | None = None): + self.settings_dict = settings_dict or {} + + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) + + class MockSpider: + def __init__(self) -> None: + self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False}) + + spider = MockSpider() + + def gen_with_return(): + yield 1 + return "value" + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + warn_on_generator_with_return_value(spider, gen_with_return) # type: ignore[arg-type] + + spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True + + with pytest.warns(UserWarning, match="is a generator"): + warn_on_generator_with_return_value(spider, gen_with_return) # type: ignore[arg-type] diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 5333a55cb..7eade6463 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -1,14 +1,20 @@ +from __future__ import annotations + import os from pathlib import Path +from typing import TYPE_CHECKING import pytest from scrapy.utils.misc import set_environ from scrapy.utils.project import data_path, get_project_settings +if TYPE_CHECKING: + from collections.abc import Generator + @pytest.fixture -def proj_path(tmp_path): +def proj_path(tmp_path: Path) -> Generator[Path]: prev_dir = Path.cwd() project_dir = tmp_path @@ -21,7 +27,7 @@ def proj_path(tmp_path): os.chdir(prev_dir) -def test_data_path_outside_project(): +def test_data_path_outside_project() -> None: assert str(Path(".scrapy", "somepath")) == data_path("somepath") abspath = str(Path(os.path.sep, "absolute", "path")) assert abspath == data_path(abspath) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 2e8047e2d..c3b5dfc99 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -22,8 +22,7 @@ from scrapy.utils.python import ( from tests.utils.decorators import coroutine_test if TYPE_CHECKING: - from collections.abc import Iterable, Mapping - + from collections.abc import AsyncIterator, Iterable, Mapping _KT = TypeVar("_KT") _VT = TypeVar("_VT") @@ -31,22 +30,22 @@ _VT = TypeVar("_VT") class TestMutableAsyncChain: @staticmethod - async def g1(): + async def g1() -> AsyncIterator[int]: for i in range(3): yield i @staticmethod - async def g2(): + async def g2() -> AsyncIterator[int]: return yield @staticmethod - async def g3(): + async def g3() -> AsyncIterator[int]: for i in range(7, 10): yield i @staticmethod - async def g4(): + async def g4() -> AsyncIterator[int]: for i in range(3, 5): yield i 1 / 0 @@ -85,7 +84,7 @@ class TestToUnicode: def test_converting_a_strange_object_should_raise_type_error(self): with pytest.raises(TypeError): - to_unicode(423) + to_unicode(423) # type: ignore[arg-type] def test_errors_argument(self): assert to_unicode(b"a\xedb", "utf-8", errors="replace") == "a\ufffdb" @@ -103,7 +102,7 @@ class TestToBytes: def test_converting_a_strange_object_should_raise_type_error(self): with pytest.raises(TypeError): - to_bytes(pytest) + to_bytes(pytest) # type: ignore[arg-type] def test_errors_argument(self): assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b" @@ -112,10 +111,10 @@ class TestToBytes: def test_memoizemethod_noargs(): class A: @memoizemethod_noargs - def cached(self): + def cached(self) -> object: return object() - def noncached(self): + def noncached(self) -> object: return object() a = A() @@ -150,7 +149,7 @@ def test_get_func_args(): pass class A: - def __init__(self, a, b, c): + def __init__(self, a: int, b: int, c: int): pass def method(self, a, b, c): diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py index 7d39a478e..44cb5c306 100644 --- a/tests/test_utils_reactor.py +++ b/tests/test_utils_reactor.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import pytest diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 1642a932b..eb95be06c 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -2,7 +2,7 @@ from __future__ import annotations import json from hashlib import sha1 -from typing import Any +from typing import TYPE_CHECKING, Any, Protocol from weakref import WeakKeyDictionary import pytest @@ -17,6 +17,9 @@ from scrapy.utils.request import ( ) from scrapy.utils.test import get_crawler +if TYPE_CHECKING: + from collections.abc import Iterable + @pytest.mark.parametrize( ("r", "expected"), @@ -56,8 +59,18 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None: request_httprepr(r) +class _FingerprintFunction(Protocol): + def __call__( + self, + request: Request, + *, + include_headers: Iterable[bytes | str] | None = None, + keep_fragments: bool = False, + ) -> bytes: ... + + class TestFingerprint: - function: staticmethod[[Request], bytes] = staticmethod(fingerprint) + function: _FingerprintFunction = staticmethod(fingerprint) cache: ( WeakKeyDictionary[ Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes] @@ -261,6 +274,7 @@ class TestRequestFingerprinter: def test_fingerprint(self): crawler = get_crawler() request = Request("https://example.com") + assert crawler.request_fingerprinter assert crawler.request_fingerprinter.fingerprint(request) == fingerprint( request ) @@ -277,6 +291,7 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) + assert crawler.request_fingerprinter r1 = Request("http://www.example.com", headers={"X-ID": "1"}) fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", headers={"X-ID": "2"}) @@ -285,9 +300,9 @@ class TestCustomRequestFingerprinter: def test_dont_canonicalize(self): class RequestFingerprinter: - cache = WeakKeyDictionary() + cache: WeakKeyDictionary[Request, bytes] = WeakKeyDictionary() - def fingerprint(self, request): + def fingerprint(self, request: Request) -> bytes: if request not in self.cache: fp = sha1() fp.update(to_bytes(request.url)) @@ -299,6 +314,7 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) + assert crawler.request_fingerprinter r1 = Request("http://www.example.com?a=1&a=2") fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com?a=2&a=1") @@ -317,6 +333,7 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) + assert crawler.request_fingerprinter r1 = Request("http://www.example.com") fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", meta={"fingerprint": "a"}) @@ -348,6 +365,7 @@ class TestCustomRequestFingerprinter: } crawler = get_crawler(settings_dict=settings) + assert crawler.request_fingerprinter request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) assert fingerprint == settings["FINGERPRINT"] diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 146cdb802..608b2bbd9 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from pathlib import Path from time import process_time from urllib.parse import urlparse @@ -15,7 +17,7 @@ from scrapy.utils.response import ( ) -def _read_browser_output(burl: str): +def _read_browser_output(burl: str) -> bytes: path = urlparse(burl).path if not path or not Path(path).exists(): path = burl.replace("file://", "") @@ -224,7 +226,7 @@ def test_open_in_browser_redos_head(): (b"real", b"real"), ], ) -def test_remove_html_comments(input_body, output_body): +def test_remove_html_comments(input_body: bytes, output_body: bytes) -> None: assert _remove_html_comments(input_body) == output_body diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index 2e6a790f8..2702c2cce 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import dataclasses import datetime import json diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index dbb9caf2a..3109bc149 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -21,9 +21,6 @@ from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from collections.abc import Callable -if TYPE_CHECKING: - from collections.abc import Callable - class TestSendCatchLog: # whether the function being tested returns exceptions or failures diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index ac57e1739..9f2dce6d6 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from scrapy.exceptions import ScrapyDeprecationWarning diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 4515ce36e..076edf3db 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,7 +1,14 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + from scrapy.utils.template import render_templatefile +if TYPE_CHECKING: + from pathlib import Path -def test_simple_render(tmp_path): + +def test_simple_render(tmp_path: Path) -> None: context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"} template = "from ${project_name}.spiders.${name} import ${classname}" rendered = "from proj.spiders.spi import TheSpider" diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 2334c76e9..5458aa603 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import sys from io import StringIO from unittest import mock @@ -46,13 +48,13 @@ Bar 1 oldest: 0s ago @mock.patch("sys.stdout", new_callable=StringIO) -def test_print_live_refs_empty(stdout): +def test_print_live_refs_empty(stdout: StringIO) -> None: trackref.print_live_refs() assert stdout.getvalue() == "Live References\n\n\n" @mock.patch("sys.stdout", new_callable=StringIO) -def test_print_live_refs_with_objects(stdout): +def test_print_live_refs_with_objects(stdout: StringIO) -> None: o1 = Foo() # noqa: F841 trackref.print_live_refs() assert ( diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 5b98131a1..d9d162c23 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from scrapy.linkextractors import IGNORED_EXTENSIONS @@ -191,7 +193,7 @@ def test_guess_scheme(url: str, expected: str): ), ], ) -def test_guess_scheme_skipped(url: str, expected: str, reason: str): +def test_guess_scheme_skipped(url: str, expected: str, reason: str) -> None: pytest.skip(reason) diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index a431091d5..4ff562dff 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -16,7 +16,7 @@ class MyRequest2(Request): @pytest.mark.mypy_testing def mypy_test_headers() -> None: - Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None" Request("data:,", headers=None) Request("data:,", headers={}) Request("data:,", headers=[]) diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index d497c2470..1c157328c 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse @pytest.mark.mypy_testing def mypy_test_headers() -> None: - Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None" Response("data:,", headers=None) Response("data:,", headers={}) Response("data:,", headers=[]) From bc5b5fb1f6d8fef2f57ef74a728dd609b5c00cf4 Mon Sep 17 00:00:00 2001 From: Youssef Mohamed <114195599+MegumiinUwU@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:47:15 +0300 Subject: [PATCH 017/111] Add Request.to_curl() (#7743) (#7802) --- docs/topics/request-response.rst | 2 ++ scrapy/http/request/__init__.py | 14 ++++++++++++++ tests/test_utils_request.py | 11 +++++++++++ tests/utils/bases/http_request.py | 9 +++++++++ 4 files changed, 36 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index b83a04032..8e565907f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -260,6 +260,8 @@ Request objects .. automethod:: from_curl + .. automethod:: to_curl + .. automethod:: to_dict diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 57517a65b..68847283a 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -387,6 +387,20 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) + def to_curl(self) -> str: + """Return a string with a `cURL `_ command equivalent + to this request. + + Inverse of :meth:`from_curl`. See also + :func:`scrapy.utils.request.request_to_curl`. + + .. versionadded:: VERSION + """ + # Imported here to avoid a circular import. + from scrapy.utils.request import request_to_curl # noqa: PLC0415 + + return request_to_curl(self) + def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]: """Return a dictionary containing the Request's data. diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index eb95be06c..935447bc4 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -475,3 +475,14 @@ class TestRequestToCurl: " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=1'" ) self._test_request(request_object, expected_curl_command) + + def test_request_to_curl_method(self) -> None: + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + 'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\'' + ) + assert request_object.to_curl() == expected_curl_command diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.py index c255b5e4c..3a1e588ef 100644 --- a/tests/utils/bases/http_request.py +++ b/tests/utils/bases/http_request.py @@ -6,6 +6,7 @@ import pytest from scrapy.http import Headers, Request from scrapy.http.request import NO_CALLBACK +from scrapy.utils.request import request_to_curl class TestRequestBase(ABC): @@ -488,3 +489,11 @@ class TestRequestBase(ABC): 'curl -X PATCH "http://example.org" --foo -z', ignore_unknown_options=False, ) + + def test_to_curl(self): + # Note: more curated tests regarding curl conversion are in + # `test_utils_request.py` + r = self.request_class( + "http://www.example.com/", method="POST", body=b"foo=bar" + ) + assert r.to_curl() == request_to_curl(r) From 8b5147ae2ec62bca66e3b957a8d0f1c3654ecf2e Mon Sep 17 00:00:00 2001 From: Adrian Date: Wed, 29 Jul 2026 11:32:18 +0200 Subject: [PATCH 018/111] Improve test coverage for scrapy.pipelines (#7798) * Improve test coverage for scrapy.pipelines * Restore old Pillow support --- tests/test_pipeline_files.py | 220 ++++++++++++++++++++++++++++++--- tests/test_pipeline_images.py | 52 +++++++- tests/test_pipeline_media.py | 60 +++++++++ tests/utils/media_pipelines.py | 6 + 4 files changed, 321 insertions(+), 17 deletions(-) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4f6fa21a0..97daa514e 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -33,7 +33,7 @@ from scrapy.pipelines.files import ( GCSFilesStore, S3FilesStore, ) -from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered +from scrapy.pipelines.media import _MediaRequestFiltered from scrapy.settings import Settings from scrapy.utils.asyncio import call_later from scrapy.utils.defer import maybe_deferred_to_future @@ -43,11 +43,7 @@ from tests.mockserver.ftp import MockFTPServer from tests.utils.decorators import coroutine_test, inline_callbacks_test from .utils.cloud import mock_google_cloud_storage -from .utils.media_pipelines import mocked_download_func - -# required by persist_file() and stat_file(), but as some stores don't use the argument -# we can pass this singleton to keep type hints correct -DUMMY_SPIDER_INFO = MediaPipeline.SpiderInfo(DefaultSpider()) +from .utils.media_pipelines import DUMMY_SPIDER_INFO, mocked_download_func def get_ftp_content_and_delete( @@ -94,16 +90,19 @@ class DeferredFSFilesStore(FSFilesStore): class TestFilesPipeline: def setup_method(self): self.tempdir = mkdtemp() - settings_dict = {"FILES_STORE": self.tempdir} - crawler = get_crawler(DefaultSpider, settings_dict=settings_dict) - crawler.spider = crawler._create_spider() - crawler.engine = MagicMock(download_async=mocked_download_func) - self.pipeline = FilesPipeline.from_crawler(crawler) - self.pipeline.open_spider() + self.pipeline = self._create_pipeline(FilesPipeline) def teardown_method(self): rmtree(self.tempdir) + def _create_pipeline(self, pipeline_cls: type[FilesPipeline]) -> FilesPipeline: + crawler = get_crawler(DefaultSpider, {"FILES_STORE": self.tempdir}) + crawler.spider = crawler._create_spider() + crawler.engine = MagicMock(download_async=mocked_download_func) + pipeline = pipeline_cls.from_crawler(crawler) + pipeline.open_spider() + return pipeline + def test_file_path_query_parameters(self): file_path = self.pipeline.file_path @@ -254,6 +253,107 @@ class TestFilesPipeline: assert result["files"][0]["checksum"] != "abc" assert result["files"][0]["status"] == "cached" + @coroutine_test + async def test_file_stat_without_last_modified(self) -> None: + """A stat result without a last modification time forces a download.""" + item_url = "http://example.com/file4.pdf" + item = _create_item_with_files(item_url) + with ( + mock.patch.object(FilesPipeline, "inc_stats", return_value=True), + mock.patch.object( + FSFilesStore, "stat_file", return_value={"checksum": "abc"} + ), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + ): + result = await self.pipeline.process_item(item) + assert result["files"][0]["checksum"] != "abc" + assert result["files"][0]["status"] == "downloaded" + + @coroutine_test + async def test_file_empty_content(self, caplog: pytest.LogCaptureFixture) -> None: + item_url = "http://example.com/empty.pdf" + item = _create_item_with_files(item_url) + request = Request( + item_url, meta={"response": Response(item_url, status=200, body=b"")} + ) + with ( + caplog.at_level(logging.WARNING), + mock.patch.object( + FilesPipeline, "get_media_requests", return_value=[request] + ), + ): + result = await self.pipeline.process_item(item) + assert result["files"] == [] + assert "File (empty-content): Empty file from" in caplog.text + + @coroutine_test + async def test_file_downloaded_file_exception( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """A FileException from file_downloaded() is logged as a warning and + kept as is.""" + + class FailingFilesPipeline(FilesPipeline): + def file_downloaded(self, response, request, info, *, item=None): + raise FileException("boom") + + item_url = "http://example.com/file5.pdf" + item = _create_item_with_files(item_url) + pipeline = self._create_pipeline(FailingFilesPipeline) + with ( + caplog.at_level(logging.WARNING), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + ): + result = await pipeline.process_item(item) + assert result["files"] == [] + records = [ + r for r in caplog.records if "Error processing file" in r.getMessage() + ] + assert len(records) == 1 + assert records[0].levelname == "WARNING" + assert "boom" in records[0].getMessage() + + @coroutine_test + async def test_file_downloaded_unknown_error( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Any other exception from file_downloaded() is logged as an error and + reported as a FileException.""" + + class FailingFilesPipeline(FilesPipeline): + def file_downloaded(self, response, request, info, *, item=None): + raise RuntimeError("boom") + + item_url = "http://example.com/file6.pdf" + item = _create_item_with_files(item_url) + pipeline = self._create_pipeline(FailingFilesPipeline) + with ( + caplog.at_level(logging.WARNING), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + ): + result = await pipeline.process_item(item) + assert result["files"] == [] + records = [ + r for r in caplog.records if "Error processing file" in r.getMessage() + ] + assert len(records) == 1 + assert records[0].levelname == "ERROR" + exc_info = records[0].exc_info + assert exc_info is not None + assert exc_info[0] is RuntimeError + @coroutine_test async def test_async_store(self) -> None: """Test that async persist_file() works and is awaited.""" @@ -648,9 +748,24 @@ class TestFilesPipelineCustomSettings: request = Request("http://example.com/image01.jpg") assert pipeline.file_path(request) == Path("subdir/image01.jpg") - def test_files_store_constructor_with_pathlike_object(self, tmp_path): - fs_store = FSFilesStore(tmp_path) - assert fs_store.basedir == str(tmp_path) + +class TestFSFilesStore: + def test_constructor_with_pathlike_object(self, tmp_path: Path) -> None: + assert FSFilesStore(tmp_path).basedir == str(tmp_path) + + def test_constructor_with_uri(self, tmp_path: Path) -> None: + assert FSFilesStore(f"file://{tmp_path}").basedir == str(tmp_path) + + def test_stat_file(self, tmp_path: Path) -> None: + store = FSFilesStore(tmp_path) + store.persist_file("full/filename", BytesIO(b"data"), DUMMY_SPIDER_INFO) + stat = store.stat_file("full/filename", DUMMY_SPIDER_INFO) + assert stat["checksum"] == "8d777f385d3dfec8815d20f7496026dc" + assert stat["last_modified"] == pytest.approx(time.time(), abs=60) + + def test_stat_missing_file(self, tmp_path: Path) -> None: + store = FSFilesStore(tmp_path) + assert store.stat_file("full/filename", DUMMY_SPIDER_INFO) == {} @pytest.mark.requires_botocore @@ -695,6 +810,59 @@ class TestS3FilesStore: # The call to read does not happen with Stubber assert buffer.method_calls == [mock.call.seek(0)] + @inline_callbacks_test + def test_persist_without_headers(self): + """Without custom headers only the default ones are sent.""" + bucket = "mybucket" + key = "export.csv" + buffer = mock.MagicMock() + + store = S3FilesStore(f"s3://{bucket}/{key}") + from botocore.stub import Stubber # noqa: PLC0415 + + with Stubber(store.s3_client) as stub: + stub.add_response( + "put_object", + expected_params={ + "ACL": S3FilesStore.POLICY, + "Body": buffer, + "Bucket": bucket, + "CacheControl": S3FilesStore.HEADERS["Cache-Control"], + "Key": key, + "Metadata": {}, + }, + service_response={}, + ) + + yield store.persist_file("", buffer, info=DUMMY_SPIDER_INFO) + + stub.assert_no_pending_responses() + + def test_missing_botocore(self): + with ( + mock.patch( + "scrapy.pipelines.files.is_botocore_available", return_value=False + ), + pytest.raises(NotConfigured, match="missing botocore library"), + ): + S3FilesStore("s3://mybucket/key") + + def test_wrong_uri_scheme(self): + with pytest.raises( + ValueError, + match=re.escape( + "Incorrect URI scheme in ftp://mybucket/key, expected 's3'" + ), + ): + S3FilesStore("ftp://mybucket/key") + + def test_unsupported_header(self): + store = S3FilesStore("s3://mybucket/key") + with pytest.raises( + TypeError, match='Header "X-Custom" is not supported by botocore' + ): + store._headers_to_botocore_kwargs({"X-Custom": "value"}) + @inline_callbacks_test def test_stat(self): bucket = "mybucket" @@ -901,6 +1069,28 @@ class TestFTPFileStore: ) assert data == content + @inline_callbacks_test + def test_persist_active_mode(self, monkeypatch: pytest.MonkeyPatch): + data = b"active mode" + path = "full/filename" + monkeypatch.setattr(FTPFilesStore, "FTP_USERNAME", "anonymous") + monkeypatch.setattr(FTPFilesStore, "FTP_PASSWORD", "guest") + monkeypatch.setattr(FTPFilesStore, "USE_ACTIVE_MODE", True) + with MockFTPServer() as ftp_server: + store = FTPFilesStore(ftp_server.url("/")) + yield store.persist_file(path, BytesIO(data), info=DUMMY_SPIDER_INFO) + stat = yield store.stat_file(path, info=DUMMY_SPIDER_INFO) + assert stat["checksum"] == "ff1575649a39a27c13faa0d37c84bab3" + + def test_wrong_uri_scheme(self): + with pytest.raises( + ValueError, + match=re.escape( + "Incorrect URI scheme in http://example.com/, expected 'ftp'" + ), + ): + FTPFilesStore("http://example.com/") + class ItemWithFiles(Item): file_urls = Field() diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 1b73dd157..19e61579f 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -3,20 +3,26 @@ from __future__ import annotations import dataclasses import io import random +import sys from abc import ABC, abstractmethod +from pathlib import Path from shutil import rmtree from tempfile import mkdtemp +from types import SimpleNamespace from typing import Any import attr import pytest from itemadapter import ItemAdapter +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.item import Field, Item -from scrapy.pipelines.files import GCSFilesStore, S3FilesStore +from scrapy.pipelines.files import GCSFilesStore, S3FilesStore, _md5sum from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test +from tests.utils.media_pipelines import DUMMY_SPIDER_INFO try: from PIL import Image @@ -40,6 +46,11 @@ class TestImagesPipeline: def teardown_method(self): rmtree(self.tempdir) + def test_missing_pillow(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "PIL", None) + with pytest.raises(NotConfigured, match="requires installing Pillow"): + ImagesPipeline(self.tempdir, crawler=get_crawler()) + def test_file_path(self): file_path = self.pipeline.file_path assert ( @@ -197,6 +208,25 @@ class TestImagesPipeline: assert path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" assert new_im.getpixel((0, 0)) == (255, 0, 0) + @coroutine_test + async def test_image_downloaded(self) -> None: + """The image and its thumbnails are stored, and the checksum of the + full-size image is returned.""" + self.pipeline.thumbs = {"small": (20, 20)} + _, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) + url = "https://dev.mydeco.com/mydeco.gif" + response = Response(url=url, body=buf.getvalue()) + + checksum = await self.pipeline.image_downloaded( + response, Request(url=url), DUMMY_SPIDER_INFO + ) + + buf.seek(0) + assert checksum == _md5sum(buf) + name = "3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" + assert Path(self.tempdir, "full", name).read_bytes() == buf.getvalue() + assert Path(self.tempdir, "thumbs", "small", name).exists() + def test_convert_image(self): SIZE = (100, 100) # straight forward case: RGB and JPEG @@ -230,6 +260,24 @@ class TestImagesPipeline: assert converted.mode == "RGB" assert converted.getcolors() == [(10000, (205, 230, 255))] + def test_convert_image_legacy_resampling_filter( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Pillow older than 9.1.0 has Image.ANTIALIAS instead of + Image.Resampling.LANCZOS.""" + # Image.LANCZOS is the only spelling that exists in every supported + # Pillow version, but Pillow defines it dynamically, hence the ignore. + monkeypatch.setattr( + self.pipeline, + "_Image", + SimpleNamespace(ANTIALIAS=Image.LANCZOS), # type: ignore[attr-defined] + ) + im, buf = _create_image("JPEG", "RGB", (100, 100), (0, 127, 255)) + + thumbnail, _ = self.pipeline.convert_image(im, size=(10, 25), response_body=buf) + + assert thumbnail.size == (10, 10) + @pytest.mark.parametrize( "bad_type", [ @@ -581,7 +629,7 @@ class TestImagesPipelineCustomSettings: GCSFilesStore.POLICY = old_policy -def _create_image(format_, *a, **kw): +def _create_image(format_: str, *a: Any, **kw: Any) -> tuple[Image.Image, io.BytesIO]: buf = io.BytesIO() Image.new(*a, **kw).save(buf, format_) buf.seek(0) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 23c19e4be..ba1c18006 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -319,6 +319,42 @@ class TestMediaPipeline(TestBaseMediaPipeline): assert self.fingerprint(req1) == self.fingerprint(req2) assert new_item["results"] == [(True, {})] + @coroutine_test + async def test_failures_are_cached_across_multiple_items(self): + self.pipe.LOG_FAILED_RESULTS = False + exc = Exception("foo") + req1 = Request("http://url1", meta={"response": exc}) + new_item = await self.pipe.process_item({"requests": req1}) + assert new_item["results"][0][1].value is exc + + # rsp2 is ignored, the cached failure must be reused because request + # fingerprints are the same + req2 = Request( + req1.url, meta={"response": Response("http://donot.download.me")} + ) + new_item = await self.pipe.process_item({"requests": req2}) + assert new_item["results"][0][0] is False + assert new_item["results"][0][1].value is exc + assert self.pipe._mockcalled.count("media_to_download") == 1 + + @coroutine_test + async def test_cached_failure_calls_errback(self): + """The errback of a request is called for a cached failure as well.""" + self.pipe.LOG_FAILED_RESULTS = False + exc = Exception("foo") + await self.pipe.process_item( + {"requests": Request("http://url1", meta={"response": exc})} + ) + + def errback(failure): + self.pipe._mockcalled.append("request_errback") + return {"recovered": failure.value} + + req = Request("http://url1", errback=errback) + new_item = await self.pipe.process_item({"requests": req}) + assert new_item["results"] == [(True, {"recovered": exc})] + assert self.pipe._mockcalled.count("request_errback") == 1 + @coroutine_test async def test_results_are_cached_for_requests_of_single_item(self): rsp1 = Response("http://url1") @@ -472,6 +508,30 @@ class TestBuildFromCrawler: assert pipe._from_crawler_called +class MediaFailedNonePipeline(MockedMediaPipeline): + def media_failed(self, failure, request, info): + self._mockcalled.append("media_failed") + + +class TestMediaFailedNone(TestBaseMediaPipeline): + """Test what happens when media_failed() neither raises an exception nor + returns a failure.""" + + pipeline_class = MediaFailedNonePipeline + + @coroutine_test + async def test_result_none(self): + req = Request("http://url1", meta={"response": Exception("foo")}) + new_item = await self.pipe.process_item({"requests": req}) + assert new_item["results"] == [(True, None)] + assert self.pipe._mockcalled == [ + "get_media_requests", + "media_to_download", + "media_failed", + "item_completed", + ] + + class MediaFailedFailurePipeline(MockedMediaPipeline): def media_failed(self, failure, request, info): self._mockcalled.append("media_failed") diff --git a/tests/utils/media_pipelines.py b/tests/utils/media_pipelines.py index 283c95940..7e013b797 100644 --- a/tests/utils/media_pipelines.py +++ b/tests/utils/media_pipelines.py @@ -3,6 +3,12 @@ from __future__ import annotations from typing import Any from scrapy.http.request import NO_CALLBACK, Request +from scrapy.pipelines.media import MediaPipeline +from scrapy.utils.spider import DefaultSpider + +# required by persist_file() and stat_file(), but as some stores don't use the argument +# we can pass this singleton to keep type hints correct +DUMMY_SPIDER_INFO = MediaPipeline.SpiderInfo(DefaultSpider()) async def mocked_download_func(request: Request) -> Any: From 0cbb20e8e83996a4d8a3339254c2d7e9b763ce7f Mon Sep 17 00:00:00 2001 From: Adrian Date: Wed, 29 Jul 2026 11:39:27 +0200 Subject: [PATCH 019/111] Treat broken cache records as cache misses (#7805) --- docs/topics/downloader-middleware.rst | 4 + scrapy/downloadermiddlewares/httpcache.py | 21 +++++- tests/test_downloadermiddleware_httpcache.py | 77 +++++++++++++++++--- 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 934eb19ac..fcfe7fd29 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -564,6 +564,10 @@ defines the methods described below. Return response if present in cache, or ``None`` otherwise. + If this method raises an exception, e.g. because the cache entry is + corrupted, the middleware logs a warning and handles the request as a + cache miss. + :param spider: the spider which generated the request :type spider: :class:`~scrapy.Spider` object diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index c6c811809..e7ca0ac0e 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from email.utils import formatdate from typing import TYPE_CHECKING @@ -28,6 +29,9 @@ if TYPE_CHECKING: from scrapy.statscollectors import StatsCollector +logger = logging.getLogger(__name__) + + class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = ( ConnectionDone, @@ -77,9 +81,20 @@ class HttpCacheMiddleware: return None # Look for cached response and check if expired - cachedresponse: Response | None = self.storage.retrieve_response( - self.crawler.spider, request - ) + cachedresponse: Response | None + try: + cachedresponse = self.storage.retrieve_response( + self.crawler.spider, request + ) + except Exception: + self.stats.inc_value("httpcache/retrieve_error") + logger.warning( + f"Could not read the cache entry for {request}, treating it as a " + f"cache miss.", + exc_info=True, + extra={"spider": self.crawler.spider}, + ) + cachedresponse = None if cachedresponse is None: self.stats.inc_value("httpcache/miss") if self.ignore_missing: diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9d5d6874e..ce56ee11d 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -1,10 +1,12 @@ from __future__ import annotations import email.utils +import logging import shutil import tempfile import time from contextlib import contextmanager +from pathlib import Path from typing import TYPE_CHECKING, Any from unittest import mock @@ -93,6 +95,12 @@ class TestBase: class StorageTestMixin: """Mixin containing storage-specific test methods.""" + def _corrupt_cache_entry( + self, storage: Any, spider: Spider, request: Request + ) -> None: + """Make the cache entry of *request* unreadable for *storage*.""" + raise NotImplementedError + def test_storage(self): with self._storage(HTTPCACHE_EXPIRATION_SECS=1) as (storage, crawler): request2 = self.request.copy() @@ -115,6 +123,38 @@ class StorageTestMixin: with mock.patch("scrapy.extensions.httpcache.time", return_value=future): assert storage.retrieve_response(crawler.spider, self.request) + def test_corrupted_cache_entry_is_a_miss(self, caplog): + with self._middleware() as mw: + spider = mw.crawler.spider + mw.storage.store_response(spider, self.request, self.response) + self._corrupt_cache_entry(mw.storage, spider, self.request) + + caplog.clear() + with caplog.at_level(logging.WARNING): + assert mw.process_request(self.request) is None + + assert "treating it as a cache miss" in caplog.text + assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1 + assert mw.crawler.stats.get_value("httpcache/miss") == 1 + + # Storing the response again replaces the corrupted cache entry. + mw.storage.store_response(spider, self.request, self.response) + self.assertEqualResponse( + self.response, mw.storage.retrieve_response(spider, self.request) + ) + + def test_corrupted_cache_entry_ignore_missing(self): + with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: + spider = mw.crawler.spider + mw.storage.store_response(spider, self.request, self.response) + self._corrupt_cache_entry(mw.storage, spider, self.request) + + with pytest.raises(IgnoreRequest): + mw.process_request(self.request) + + assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1 + assert mw.crawler.stats.get_value("httpcache/ignore") == 1 + def test_storage_no_content_type_header(self): """Test that the response body is used to get the right response class even if there is no Content-Type header""" @@ -556,29 +596,43 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): # Concrete test classes that combine storage and policy mixins -class TestFilesystemStorageWithDummyPolicy( - TestBase, StorageTestMixin, DummyPolicyTestMixin -): +class FilesystemStorageTestMixin(StorageTestMixin): storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" + + def _corrupt_cache_entry(self, storage, spider, request) -> None: + rpath = Path(storage._get_request_path(spider, request)) + (rpath / "response_body").unlink() + + +class DbmStorageTestMixin(StorageTestMixin): + storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" + + def _corrupt_cache_entry(self, storage, spider, request) -> None: + key = storage._fingerprinter.fingerprint(request).hex() + storage.db[f"{key}_data"] = b"not a pickle" + + +class TestFilesystemStorageWithDummyPolicy( + TestBase, FilesystemStorageTestMixin, DummyPolicyTestMixin +): policy_class = "scrapy.extensions.httpcache.DummyPolicy" class TestFilesystemStorageWithRFC2616Policy( - TestBase, StorageTestMixin, RFC2616PolicyTestMixin + TestBase, FilesystemStorageTestMixin, RFC2616PolicyTestMixin ): - storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" policy_class = "scrapy.extensions.httpcache.RFC2616Policy" -class TestDbmStorageWithDummyPolicy(TestBase, StorageTestMixin, DummyPolicyTestMixin): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" +class TestDbmStorageWithDummyPolicy( + TestBase, DbmStorageTestMixin, DummyPolicyTestMixin +): policy_class = "scrapy.extensions.httpcache.DummyPolicy" class TestDbmStorageWithRFC2616Policy( - TestBase, StorageTestMixin, RFC2616PolicyTestMixin + TestBase, DbmStorageTestMixin, RFC2616PolicyTestMixin ): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" policy_class = "scrapy.extensions.httpcache.RFC2616Policy" @@ -599,3 +653,8 @@ class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPol def _get_settings(self, **new_settings) -> dict[str, Any]: new_settings.setdefault("HTTPCACHE_GZIP", True) return super()._get_settings(**new_settings) + + def _corrupt_cache_entry(self, storage, spider, request) -> None: + # A spider killed while writing a gzip file leaves it truncated. + body_path = Path(storage._get_request_path(spider, request), "response_body") + body_path.write_bytes(body_path.read_bytes()[:-5]) From 01447f996500367fb999560d08a40cd2ff68815a Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 29 Jul 2026 08:00:24 -0300 Subject: [PATCH 020/111] fix(commands/parse): Restore request callback before invoking spider (#7803) * fix(commands/parse): restore request callback before invoking spider * Add other test --- scrapy/commands/parse.py | 2 ++ tests/test_command_parse.py | 56 +++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 2ac65bf3f..93194ded7 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -346,6 +346,8 @@ class Command(BaseRunSpiderCommand): self.first_response = response cb = self._get_callback(spider=spider, opts=opts, response=response) + assert response.request + response.request.callback = cb # parse items and requests depth: int = response.meta["_depth"] diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 9b7131c7a..772cc82e2 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -126,6 +126,32 @@ class MySpider(scrapy.Spider): else: self.logger.debug('It Does Not Work :(') +class RetryRequestSpider(BaseSpider): + name = 'retry_request' + + def parse(self, response): + if response.meta.get('retried'): + yield {{'retried': True}} + return + response.meta['retried'] = True + yield response.request.replace(dont_filter=True) + +class CustomCallbackRetryRequestSpider(BaseSpider): + name = 'retry_request_custom_callback' + + def parse(self, response): + yield response.request.replace( + callback=self.parse_retry, + dont_filter=True, + ) + + def parse_retry(self, response): + if response.meta.get('retried'): + yield {{'retried_with_custom_callback': True}} + return + response.meta['retried'] = True + yield response.request.replace(dont_filter=True) + class MyGoodCrawlSpider(CrawlSpider): name = 'goodcrawl{self.spider_name}' @@ -381,6 +407,36 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "[{}, {'foo': 'bar'}]" in out + def test_retry_response_request( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, out, stderr = proc( + "parse", + "--spider", + "retry_request", + "-d", + "2", + mockserver.url("/html"), + cwd=proj_path, + ) + assert "RecursionError" not in stderr + assert "{'retried': True}" in out + + def test_retry_response_request_with_custom_callback( + self, proj_path: Path, mockserver: MockServer + ) -> None: + _, out, stderr = proc( + "parse", + "--spider", + "retry_request_custom_callback", + "-d", + "3", + mockserver.url("/html"), + cwd=proj_path, + ) + assert "RecursionError" not in stderr + assert "{'retried_with_custom_callback': True}" in out + def test_wrong_callback_passed( self, proj_path: Path, mockserver: MockServer ) -> None: From b2d4eedea8873700b2597a44dabafe7c9d169275 Mon Sep 17 00:00:00 2001 From: Fandu <113630375+mrfandu1@users.noreply.github.com> Date: Wed, 29 Jul 2026 20:15:03 +0545 Subject: [PATCH 021/111] Fix immediate delivery of full feed export batches (#7730) (#7733) * Store full feed batches before spider closes (#7730) Start closing and storing each batch as soon as it reaches the configured item count. Track unfinished close tasks so spider shutdown still waits for all deliveries before emitting the exporter-closed signal. Add an end-to-end regression test that verifies the first batch is stored while the crawl is still running. * Remove the issue reference --------- Co-authored-by: Andrey Rakhmatullin --- scrapy/extensions/feedexport.py | 39 +++++++++++++++++++++++-------- tests/test_feedexport_batch.py | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 678a29e2e..c2997921d 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -13,7 +13,7 @@ import re import sys import warnings from abc import ABC, abstractmethod -from collections.abc import Callable, Coroutine +from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile @@ -475,7 +475,7 @@ class FeedExporter: self.feeds = {} self.slots: list[FeedSlot] = [] self.filters: dict[str, ItemFilter] = {} - self._pending_close_coros: list[Coroutine[Any, Any, None]] = [] + self._pending_close_tasks: list[asyncio.Task[None] | Deferred[None]] = [] if not self.settings["FEEDS"] and not self.settings["FEED_URI"]: raise NotConfigured @@ -539,23 +539,44 @@ class FeedExporter: ) async def close_spider(self, spider: Spider) -> None: - self._pending_close_coros.extend( - self._close_slot(slot, spider) for slot in self.slots - ) + for slot in self.slots: + self._schedule_slot_close(slot, spider) - if self._pending_close_coros: + if self._pending_close_tasks: if is_asyncio_available(): await asyncio.wait( - [asyncio.create_task(coro) for coro in self._pending_close_coros] + cast("list[asyncio.Task[None]]", list(self._pending_close_tasks)) ) else: await DeferredList( - deferred_from_coro(coro) for coro in self._pending_close_coros + cast("list[Deferred[None]]", list(self._pending_close_tasks)) ) # Send FEED_EXPORTER_CLOSED signal await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed) + def _schedule_slot_close( + self, slot: FeedSlot, spider: Spider + ) -> asyncio.Task[None] | Deferred[None]: + """Start closing the slot without waiting for it to finish, keeping + track of the pending work so that it can be awaited in + :meth:`close_spider` if it hasn't finished by then.""" + aw: asyncio.Task[None] | Deferred[None] + coro = self._close_slot(slot, spider) + if is_asyncio_available(): + aw = asyncio.create_task(coro) + self._pending_close_tasks.append(aw) + aw.add_done_callback(self._pending_close_tasks.remove) + else: + aw = deferred_from_coro(coro) + self._pending_close_tasks.append(aw) + aw.addBoth(self._untrack_pending_close_task, aw) + return aw + + def _untrack_pending_close_task(self, result: Any, aw: Deferred[None]) -> Any: + self._pending_close_tasks.remove(aw) + return result + @staticmethod def _get_file(slot_: FeedSlot) -> IO[bytes]: assert slot_.file @@ -652,7 +673,7 @@ class FeedExporter: uri_params = self._get_uri_params( spider, self.feeds[slot.uri_template]["uri_params"], slot ) - self._pending_close_coros.append(self._close_slot(slot, spider)) + self._schedule_slot_close(slot, spider) slots.append( self._start_new_batch( batch_id=slot.batch_id + 1, diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 80ff6229b..4b0962c43 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -210,6 +210,47 @@ class TestBatchDeliveries(TestFeedExportBase): header = MyItem.fields.keys() await self.assertExported(items, header, rows, settings=settings) + @coroutine_test + async def test_batch_delivered_when_full(self): + """Full batches must be finalized and delivered as soon as they are + full, instead of when the spider closes.""" + dir_path = self._random_temp_filename() + batch1_path = Path(dir_path, "1.json") + mockserver_url = self.mockserver.url("/") + batch1_contents: list[bytes | None] = [] + + class TestSpider(scrapy.Spider): + name = "testspider" + start_urls = [mockserver_url] + + def parse(self, response): + yield {"foo": "bar1"} + yield {"foo": "bar2"} + yield scrapy.Request( + mockserver_url, callback=self.parse2, dont_filter=True + ) + + def parse2(self, response): + # the first batch was full after the second item, so it must + # have been delivered by now + batch1_contents.append( + batch1_path.read_bytes() if batch1_path.exists() else None + ) + yield {"foo": "bar3"} + + settings = { + "FEEDS": { + build_url(dir_path / "%(batch_id)d.json"): {"format": "json"}, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, + } + crawler = get_crawler(TestSpider, settings) + await crawler.crawl_async() + + assert batch1_contents, "the second request was not processed" + assert batch1_contents[0] is not None, "batch 1 was not stored during the crawl" + assert json.loads(batch1_contents[0]) == [{"foo": "bar1"}, {"foo": "bar2"}] + def test_wrong_path(self): """If path is without %(batch_time)s and %(batch_id) an exception must be raised""" settings = { From 5b4828a012fcd136a8f46915e19be07a5029e57e Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 30 Jul 2026 03:02:54 -0300 Subject: [PATCH 022/111] docs(practices): Remove scrapoxy mention (#7817) --- docs/topics/practices.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 23738c98c..dfa1e21f6 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -533,8 +533,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites directly * use a pool of rotating IPs. For example, the free `Tor project`_ or paid - services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a - super proxy that you can attach your own proxies to. + services like `ProxyMesh`_. * for HTTPS websites, if blocking appears related to TLS behavior, consider adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond @@ -559,5 +558,4 @@ projects that detects common mistakes and anti-patterns. .. _ProxyMesh: https://proxymesh.com/ .. _Common Crawl: https://commoncrawl.org/ .. _testspiders: https://github.com/scrapinghub/testspiders -.. _scrapoxy: https://scrapoxy.io/ .. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html From 98696efa809ddde93f78cabd555db832f270553f Mon Sep 17 00:00:00 2001 From: Adrian Date: Thu, 30 Jul 2026 12:46:45 +0200 Subject: [PATCH 023/111] Export item fields in declaration order (#7824) --- docs/topics/exporters.rst | 10 ++++++++++ scrapy/exporters.py | 17 ++++++++++++++++- tests/test_exporters.py | 12 ++++++++++++ tests/test_feedexport.py | 12 ++++++------ 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index ecd154122..c43b7e20f 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -211,6 +211,16 @@ BaseItemExporter - ``None`` (all fields [2]_, default) + Fields are exported in declaration order, i.e. the order in which + they are defined in the :ref:`item class `. For + :class:`dict` items, which have no declared fields, the key order of + each item is used instead. + + .. versionchanged:: VERSION + Fields of non-\ :class:`dict` items used to be exported in the + order in which they had been populated, except in + :class:`CsvItemExporter`, which has always used declaration order. + - A list of fields: .. code-block:: python diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 5a11df833..ea600d1a8 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -74,6 +74,17 @@ class BaseItemExporter(ABC): def finish_exporting(self) -> None: # noqa: B027 pass + @staticmethod + def _get_populated_field_names(adapter: ItemAdapter) -> Iterable[str]: + """Return the populated field names of *adapter*, in declaration order. + + Populated fields that are not declared, which some item types allow, + come last, in item order. + """ + populated = set(adapter.keys()) + declared = (name for name in adapter.field_names() if name in populated) + return dict.fromkeys([*declared, *adapter.keys()]) + def _get_serialized_fields( self, item: Any, default_value: Any = None, include_empty: bool | None = None ) -> Iterable[tuple[str, Any]]: @@ -86,7 +97,11 @@ class BaseItemExporter(ABC): include_empty = self.export_empty_fields if self.fields_to_export is None: - field_iter = item.field_names() if include_empty else item.keys() + field_iter = ( + item.field_names() + if include_empty + else self._get_populated_field_names(item) + ) elif isinstance(self.fields_to_export, Mapping): if include_empty: field_iter = self.fields_to_export.items() diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 3b997767e..b857728ba 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -118,6 +118,18 @@ class TestBaseItemExporter(ABC): ie = self._get_exporter(fields_to_export={"name": "名稱"}) assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")] + def test_field_order(self): + item = self.item_class(age="22", name="John\xa3") + ie = self._get_exporter() + assert [name for name, _ in ie._get_serialized_fields(item)] == ["name", "age"] + + def test_field_order_dict_item(self): + ie = self._get_exporter() + assert [name for name, _ in ie._get_serialized_fields({"age": "22"})] == ["age"] + assert [ + name for name, _ in ie._get_serialized_fields({"age": "22", "name": "John"}) + ] == ["age", "name"] + def test_field_custom_serializer(self): i = self.custom_field_item_class(name="John\xa3", age="22") a = ItemAdapter(i) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 92c414bef..1cca287af 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -675,14 +675,14 @@ class TestFeedExport(TestFeedExportBase): formats = { "csv": b"foo,egg,baz\r\nbar1,spam1,\r\n", - "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]', + "json": b'[\n{"foo": "bar2", "hello": "world2"}\n]', "jsonlines": ( - b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n' + b'{"foo": "bar1", "egg": "spam1"}\n{"foo": "bar2", "hello": "world2"}\n' ), "xml": ( b'\n\n' - b"bar1spam1\n" - b"world2bar2\nworld3" + b"bar1spam1\n" + b"bar2world2\nworld3" b"spam3\n" ), } @@ -740,8 +740,8 @@ class TestFeedExport(TestFeedExportBase): "json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]', "xml": ( b'\n\n' - b"bar1spam1\n" - b"world2bar2\n" + b"bar1spam1\n" + b"bar2world2\n" ), "jsonlines": b'{"foo": "bar1", "egg": "spam1"}\n', } From 433603e6cab4ccea6aee18843deaa03f61e216b8 Mon Sep 17 00:00:00 2001 From: Adrian Date: Thu, 30 Jul 2026 15:51:45 +0200 Subject: [PATCH 024/111] Add AWS_MAX_POOL_CONNECTIONS (#7794) --- docs/topics/feed-exports.rst | 5 +++-- docs/topics/media-pipeline.rst | 3 +++ docs/topics/settings.rst | 20 +++++++++++++++++ scrapy/extensions/feedexport.py | 11 +++++++++ scrapy/pipelines/files.py | 13 ++++++++++- scrapy/settings/default_settings.py | 2 ++ scrapy/utils/boto.py | 15 +++++++++++++ tests/test_feedexport_storages.py | 35 +++++++++++++++++++++++++++++ tests/test_pipeline_files.py | 27 ++++++++++++++++++++++ 9 files changed, 128 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 66768c97b..2f686fd0f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -218,12 +218,13 @@ passed through the following settings: .. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html -You can also define a custom ACL, custom endpoint, and region name for exported -feeds using these settings: +You can also define a custom ACL, custom endpoint, region name and connection +pool size for exported feeds using these settings: - :setting:`FEED_STORAGE_S3_ACL` - :setting:`AWS_ENDPOINT_URL` - :setting:`AWS_REGION_NAME` +- :setting:`AWS_MAX_POOL_CONNECTIONS` The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 4ceb4732a..b16066d0c 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -268,6 +268,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S AWS_USE_SSL = False # or True (None by default) AWS_VERIFY = False # or True (None by default) +To reuse connections for as many files as you check or upload in parallel, set +:setting:`AWS_MAX_POOL_CONNECTIONS` accordingly. + .. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl .. _Minio: https://github.com/minio/minio .. _Zenko CloudServer: https://www.zenko.io/cloudserver/ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8068824e3..1b6851d04 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -458,6 +458,26 @@ Default: ``None`` Endpoint URL used for S3-like storage, for example Minio or s3.scality. +.. setting:: AWS_MAX_POOL_CONNECTIONS + +AWS_MAX_POOL_CONNECTIONS +------------------------ + +.. versionadded:: VERSION + +Default: ``None`` + +Maximum number of connections that AWS clients, such as those of the +:ref:`S3 feed storage backend ` and of the +:ref:`S3 media pipeline storage backend `, keep in their +connection pool. + +If ``None``, the value of :setting:`REACTOR_THREADPOOL_MAXSIZE` is used. + +Values lower than the number of parallel AWS calls do not limit those calls, but +their connections are closed instead of reused, which hurts performance, and +``Connection pool is full, discarding connection`` warnings are logged. + .. setting:: AWS_REGION_NAME AWS_REGION_NAME diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c2997921d..448279546 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -28,6 +28,7 @@ from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.asyncio import is_asyncio_available, run_in_thread +from scrapy.utils.boto import _get_max_pool_connections from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.ftp import ftp_store_file @@ -213,11 +214,14 @@ class S3FeedStorage(BlockingFeedStorage): feed_options: dict[str, Any] | None = None, session_token: str | None = None, region_name: str | None = None, + max_pool_connections: int | None = None, ): try: import boto3.session # noqa: PLC0415 except ImportError: raise NotConfigured("missing boto3 library") from None + from botocore.config import Config # noqa: PLC0415 + u = urlparse(uri) assert u.hostname self.bucketname: str = u.hostname @@ -228,6 +232,7 @@ class S3FeedStorage(BlockingFeedStorage): self.acl: str | None = acl self.endpoint_url: str | None = endpoint_url self.region_name: str | None = region_name + self.max_pool_connections: int | None = max_pool_connections boto3_session = boto3.session.Session() self.s3_client = boto3_session.client( @@ -237,6 +242,11 @@ class S3FeedStorage(BlockingFeedStorage): aws_session_token=self.session_token, endpoint_url=self.endpoint_url, region_name=self.region_name, + config=( + Config(max_pool_connections=self.max_pool_connections) + if self.max_pool_connections is not None + else None + ), ) if feed_options and feed_options.get("overwrite", True) is False: @@ -262,6 +272,7 @@ class S3FeedStorage(BlockingFeedStorage): acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None, endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None, region_name=crawler.settings["AWS_REGION_NAME"] or None, + max_pool_connections=_get_max_pool_connections(crawler.settings), feed_options=feed_options, ) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8e8082332..55a3676e5 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -37,7 +37,7 @@ from scrapy.pipelines.media import ( _MediaRequestFiltered, ) from scrapy.utils.asyncio import run_in_thread -from scrapy.utils.boto import is_botocore_available +from scrapy.utils.boto import _get_max_pool_connections, is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.ftp import ftp_store_file @@ -164,6 +164,9 @@ class S3FilesStore: AWS_REGION_NAME = None AWS_USE_SSL = None AWS_VERIFY = None + # Overridden from settings.AWS_MAX_POOL_CONNECTIONS in + # FilesPipeline.from_crawler(); None means the botocore default + AWS_MAX_POOL_CONNECTIONS: int | None = None POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler() HEADERS: ClassVar[dict[str, str]] = { @@ -174,7 +177,13 @@ class S3FilesStore: if not is_botocore_available(): raise NotConfigured("missing botocore library") import botocore.session # noqa: PLC0415 + from botocore.config import Config # noqa: PLC0415 + config = ( + Config(max_pool_connections=self.AWS_MAX_POOL_CONNECTIONS) + if self.AWS_MAX_POOL_CONNECTIONS is not None + else None + ) session = botocore.session.get_session() self.s3_client = session.create_client( "s3", @@ -185,6 +194,7 @@ class S3FilesStore: region_name=self.AWS_REGION_NAME, use_ssl=self.AWS_USE_SSL, verify=self.AWS_VERIFY, + config=config, ) if not uri.startswith("s3://"): raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") @@ -522,6 +532,7 @@ class FilesPipeline(MediaPipeline): s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"] s3store.AWS_USE_SSL = settings["AWS_USE_SSL"] s3store.AWS_VERIFY = settings["AWS_VERIFY"] + s3store.AWS_MAX_POOL_CONNECTIONS = _get_max_pool_connections(settings) s3store.POLICY = settings["FILES_STORE_S3_ACL"] gcs_store: type[GCSFilesStore] = cast( diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 993f2436d..a44b36c8a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -28,6 +28,7 @@ __all__ = [ "AUTOTHROTTLE_TARGET_CONCURRENCY", "AWS_ACCESS_KEY_ID", "AWS_ENDPOINT_URL", + "AWS_MAX_POOL_CONNECTIONS", "AWS_REGION_NAME", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", @@ -229,6 +230,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None AWS_ENDPOINT_URL = None +AWS_MAX_POOL_CONNECTIONS = None AWS_REGION_NAME = None AWS_SESSION_TOKEN = None AWS_USE_SSL = None diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 2a77ee2ac..76ee0e7ec 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,7 +1,22 @@ """Boto/botocore helpers""" +from __future__ import annotations + from importlib.util import find_spec +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from scrapy.settings import BaseSettings def is_botocore_available() -> bool: return find_spec("botocore") is not None + + +def _get_max_pool_connections(settings: BaseSettings) -> int: + """Return the maximum number of connections that AWS clients may keep in + their connection pool. + """ + return settings.getint("AWS_MAX_POOL_CONNECTIONS") or settings.getint( + "REACTOR_THREADPOOL_MAXSIZE" + ) diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index 66488540f..b1e3787fd 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -381,6 +381,41 @@ class TestS3FeedStorage: assert storage.region_name == region_name assert storage.s3_client._client_config.region_name == region_name + def test_init_without_max_pool_connections(self) -> None: + storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") + assert storage.max_pool_connections is None + config: Any = storage.s3_client.meta.config + assert config.max_pool_connections == 10 + + def test_init_with_max_pool_connections(self) -> None: + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + max_pool_connections=30, + ) + assert storage.max_pool_connections == 30 + config: Any = storage.s3_client.meta.config + assert config.max_pool_connections == 30 + + @pytest.mark.parametrize( + ("settings", "expected"), + [ + ({}, 10), + ({"REACTOR_THREADPOOL_MAXSIZE": 20}, 20), + ({"AWS_MAX_POOL_CONNECTIONS": 30}, 30), + ({"AWS_MAX_POOL_CONNECTIONS": 30, "REACTOR_THREADPOOL_MAXSIZE": 20}, 30), + ], + ) + def test_from_crawler_max_pool_connections( + self, settings: dict[str, Any], expected: int + ) -> None: + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + assert storage.max_pool_connections == expected + config: Any = storage.s3_client.meta.config + assert config.max_pool_connections == expected + @coroutine_test async def test_store_without_acl(self): storage = S3FeedStorage( diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 97daa514e..4e7fb118b 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -895,6 +895,33 @@ class TestS3FilesStore: stub.assert_no_pending_responses() + def test_default_max_pool_connections(self) -> None: + store = S3FilesStore("s3://mybucket/prefix/") + config: Any = store.s3_client.meta.config + assert config.max_pool_connections == 10 + + @pytest.mark.parametrize( + ("settings", "expected"), + [ + ({}, 10), + ({"REACTOR_THREADPOOL_MAXSIZE": 20}, 20), + ({"AWS_MAX_POOL_CONNECTIONS": 30}, 30), + ({"AWS_MAX_POOL_CONNECTIONS": 30, "REACTOR_THREADPOOL_MAXSIZE": 20}, 30), + ], + ) + def test_max_pool_connections( + self, monkeypatch: pytest.MonkeyPatch, settings: dict[str, Any], expected: int + ) -> None: + # restores the value that FilesPipeline.from_crawler() sets on the class + monkeypatch.setattr(S3FilesStore, "AWS_MAX_POOL_CONNECTIONS", None) + crawler = get_crawler( + settings_dict={"FILES_STORE": "s3://mybucket/prefix/", **settings} + ) + store = FilesPipeline.from_crawler(crawler).store + assert isinstance(store, S3FilesStore) + config: Any = store.s3_client.meta.config + assert config.max_pool_connections == expected + class TestGCSFilesStore: @staticmethod From f02a99fe71dd0ff2fde1ef7a533cd61904fa1c9f Mon Sep 17 00:00:00 2001 From: Adrian Date: Thu, 30 Jul 2026 17:15:15 +0200 Subject: [PATCH 025/111] Add doc sections for callbacks and errbacks (#7821) --- docs/faq.rst | 2 +- docs/intro/tutorial.rst | 2 +- docs/topics/coroutines.rst | 4 +- docs/topics/jobs.rst | 10 +- docs/topics/request-response.rst | 480 ++++++++++++++++++++----------- docs/topics/spiders.rst | 67 ++--- scrapy/http/request/__init__.py | 8 +- scrapy/spiders/__init__.py | 16 ++ 8 files changed, 372 insertions(+), 217 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 0446a6868..1a574e5da 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -97,7 +97,7 @@ handler documentation. How can I scrape an item with attributes in different pages? ------------------------------------------------------------ -See :ref:`topics-request-response-ref-request-callback-arguments`. +See :ref:`callback-data`. How can I simulate a user login in my spider? --------------------------------------------- diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index c4e04364b..eaf492c95 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -769,7 +769,7 @@ crawlers on top of it. Also, a common pattern is to build an item with data from more than one page, using a :ref:`trick to pass additional data to the callbacks -`. +`. Using spider arguments diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 9dcd9d69c..b7ddb0a57 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -21,7 +21,9 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): .. versionadded:: 2.13 -- :class:`~scrapy.Request` callbacks. +- :class:`~scrapy.Request` :ref:`callbacks `, which may + also be defined as :term:`asynchronous generators `. - The :meth:`process_item` method of :ref:`item pipelines `. diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index c9916110d..dcff10772 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -96,9 +96,13 @@ Request serialization --------------------- For persistence to work, :class:`~scrapy.Request` objects must be -serializable with :mod:`pickle`, except for the ``callback`` and ``errback`` -values passed to their ``__init__`` method, which must be methods of the -running :class:`~scrapy.Spider` class. +serializable with :mod:`pickle`, except for the :ref:`callback +` and :ref:`errback +` values passed to their ``__init__`` +method, which must be methods of the running :class:`~scrapy.Spider` class. + +Requests that cannot be serialized are kept in memory only: they are still +sent, but they are lost when the crawl is paused. If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 8e565907f..75158440b 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -205,10 +205,11 @@ Request objects Request metadata can also be accessed through the :attr:`~scrapy.http.Response.meta` attribute of a response. - To pass data from one spider callback to another, consider using - :attr:`cb_kwargs` instead. However, request metadata may be the right - choice in certain scenarios, such as to maintain some debugging data - across all follow-up requests (e.g. the source URL). + To pass your own data from one spider callback to another, use + :attr:`cb_kwargs` instead, see :ref:`callback-data`. However, request + metadata may be the right choice in certain scenarios, such as to + maintain some debugging data across all follow-up requests (e.g. the + source URL). A common use of request metadata is to define request-specific parameters for Scrapy components (extensions, middlewares, etc.). For @@ -248,7 +249,7 @@ Request objects .. method:: Request.copy() Return a new Request which is a copy of this Request. See also: - :ref:`topics-request-response-ref-request-callback-arguments`. + :ref:`callback-data`. .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs, cls]) @@ -256,7 +257,7 @@ Request objects given new values by whichever keyword arguments are specified. The :attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` attributes are shallow copied by default (unless new values are given as arguments). See also - :ref:`topics-request-response-ref-request-callback-arguments`. + :ref:`callback-data`. .. automethod:: from_curl @@ -347,160 +348,6 @@ Other functions related to requests .. autofunction:: scrapy.utils.httpobj.urlparse_cached -.. _topics-request-response-ref-request-callback-arguments: - -Passing additional data to callback functions ---------------------------------------------- - -The callback of a request is a function that will be called when the response -of that request is downloaded. The callback function will be called with the -downloaded :class:`Response` object as its first argument. - -Example: - -.. code-block:: python - - def parse_page1(self, response): - return scrapy.Request( - "http://www.example.com/some_page.html", callback=self.parse_page2 - ) - - - def parse_page2(self, response): - # this would log http://www.example.com/some_page.html - self.logger.info("Visited %s", response.url) - -In some cases you may be interested in passing arguments to those callback -functions so you can receive the arguments later, in the second callback. -The following example shows how to achieve this by using the -:attr:`.Request.cb_kwargs` attribute: - -.. code-block:: python - - def parse(self, response): - request = scrapy.Request( - "http://www.example.com/index.html", - callback=self.parse_page2, - cb_kwargs=dict(main_url=response.url), - ) - request.cb_kwargs["foo"] = "bar" # add more arguments for the callback - yield request - - - def parse_page2(self, response, main_url, foo): - yield dict( - main_url=main_url, - other_url=response.url, - foo=foo, - ) - -.. caution:: :attr:`.Request.cb_kwargs` was introduced in version ``1.7``. - Prior to that, using :attr:`.Request.meta` was recommended for passing - information around callbacks. After ``1.7``, :attr:`.Request.cb_kwargs` - became the preferred way for handling user information, leaving :attr:`.Request.meta` - for communication with components like middlewares and extensions. - -.. _topics-request-response-ref-errbacks: - -Using errbacks to catch exceptions in request processing --------------------------------------------------------- - -The errback of a request is a function that will be called when an exception -is raise while processing it. - -It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can -be used to track connection establishment timeouts, DNS errors etc. - -Here's an example spider logging all errors and catching some specific -errors if needed: - -.. code-block:: python - - import scrapy - - from scrapy.spidermiddlewares.httperror import HttpError - from twisted.internet.error import DNSLookupError - from twisted.internet.error import TimeoutError, TCPTimedOutError - - - class ErrbackSpider(scrapy.Spider): - name = "errback_example" - start_urls = [ - "http://www.httpbin.org/", # HTTP 200 expected - "http://www.httpbin.org/status/404", # Not found error - "http://www.httpbin.org/status/500", # server issue - "http://www.httpbin.org:12345/", # non-responding host, timeout expected - "https://example.invalid/", # DNS error expected - ] - - async def start(self): - for u in self.start_urls: - yield scrapy.Request( - u, - callback=self.parse_httpbin, - errback=self.errback_httpbin, - dont_filter=True, - ) - - def parse_httpbin(self, response): - self.logger.info(f"Got successful response from {response.url}") - # do something useful here... - - def errback_httpbin(self, failure): - # log all failures - self.logger.error(repr(failure)) - - # in case you want to do something special for some errors, - # you may need the failure's type: - - if failure.check(HttpError): - # these exceptions come from HttpError spider middleware - # you can get the non-200 response - response = failure.value.response - self.logger.error("HttpError on %s", response.url) - - elif failure.check(DNSLookupError): - # this is the original request - request = failure.request - self.logger.error("DNSLookupError on %s", request.url) - - elif failure.check(TimeoutError, TCPTimedOutError): - request = failure.request - self.logger.error("TimeoutError on %s", request.url) - - -.. _errback-cb_kwargs: - -Accessing additional data in errback functions ----------------------------------------------- - -In case of a failure to process the request, you may be interested in -accessing arguments to the callback functions so you can process further -based on the arguments in the errback. The following example shows how to -achieve this by using ``Failure.request.cb_kwargs``: - -.. code-block:: python - - def parse(self, response): - request = scrapy.Request( - "http://www.example.com/index.html", - callback=self.parse_page2, - errback=self.errback_page2, - cb_kwargs=dict(main_url=response.url), - ) - yield request - - - def parse_page2(self, response, main_url): - pass - - - def errback_page2(self, failure): - yield dict( - main_url=failure.request.cb_kwargs["main_url"], - ) - - .. _request-fingerprints: Request fingerprints @@ -702,6 +549,319 @@ The following built-in Scrapy components have such restrictions: 45-character-long keys must be supported. +.. _callbacks: + +Callbacks +========= + +A callback is a function that Scrapy calls with the :class:`Response` of a +:class:`~scrapy.Request` once that request has been downloaded, so that you can +extract data from that response and generate additional requests to continue +the crawl: + +.. code-block:: python + + from scrapy import Request, Spider + + + class BookSpider(Spider): + name = "books" + + async def start(self): + yield Request("https://books.toscrape.com/", callback=self.parse_home) + + def parse_home(self, response): + for url in response.css("h3 a::attr(href)").getall(): + yield Request(response.urljoin(url), callback=self.parse_book) + + def parse_book(self, response): + yield {"title": response.css("h1::text").get()} + +Requests may also define an :ref:`errback `, which Scrapy calls +instead of the callback when an exception is raised while processing the +request or its response, e.g. a connection error or, by default, a non-2xx +response. + + +.. _callback-assignment: + +Assigning a callback to a request +--------------------------------- + +To assign a callback to a request, use the ``callback`` parameter of +:class:`~scrapy.Request`, which sets the :attr:`.Request.callback` attribute: + +.. code-block:: python + + from scrapy import Request + + + def parse_home(response): ... + + + request = Request("https://books.toscrape.com/", callback=parse_home) + +Requests with no callback, i.e. with :attr:`~scrapy.Request.callback` set to +``None``, are handled by the :meth:`~scrapy.Spider.parse` method of the spider: + +.. code-block:: python + + request = Request("https://books.toscrape.com/") # Handled by parse() + +If a request is never meant to reach a spider callback, e.g. because a +:ref:`component ` sends it and handles its response itself, +assign the special :func:`~scrapy.http.request.NO_CALLBACK` value to it +instead, so that :ref:`downloader middlewares ` +can tell such requests apart. + +While :attr:`~scrapy.Request.callback` only accepts callables, some spider +classes let you also define a callback by name: both :attr:`CrawlSpider.rules +` and :attr:`SitemapSpider.sitemap_rules +` accept the name of a spider +method as a string. + + +.. _writing-callbacks: + +Writing a callback +------------------ + +Any callable can be a callback, as long as it takes the response as its first +positional parameter, and any :ref:`additional callback data ` +as keyword parameters. Spider methods are the most common choice, but plain +functions, lambda expressions and other callable objects work as well. + +.. note:: If you enable :ref:`job persistence ` through the + :setting:`JOBDIR` setting, callbacks must be methods of the running spider. + Requests with any other callback cannot be serialized, so they are kept in + memory only and lost when you pause the crawl. See + :ref:`request-serialization`. + +A callback can be: + +- A regular function: + + .. code-block:: python + + def parse(self, response): + return {"url": response.url} + +- A generator function: + + .. code-block:: python + + def parse(self, response): + yield {"url": response.url} + +- A coroutine function, i.e. defined with ``async def``: + + .. code-block:: python + + async def parse(self, response): + return {"url": response.url} + +- An asynchronous generator function: + + .. code-block:: python + + async def parse(self, response): + yield {"url": response.url} + +The last two allow using ``await``, ``async for`` and ``async with`` in your +callback. See :ref:`topics-coroutines`. + + +.. _callback-output: + +Callback output +--------------- + +A callback may return or yield any of the following: + +- ``None``, which does nothing. + + Callbacks that produce no output at all, e.g. callbacks that only log + information about the response, are perfectly valid. ``None`` values within + an iterable of callback output are ignored as well. + +- A :class:`~scrapy.Request` object, which Scrapy schedules, downloads and + eventually sends to its own callback. + +- An :ref:`item object `, which Scrapy sends to the + :ref:`item pipelines `. + + Any object that is neither ``None`` nor a :class:`~scrapy.Request` object + is treated as an item. + +- An iterable of any of the values above, e.g. a list or, more commonly, a + generator. + + :term:`Asynchronous iterables `, e.g. an + :term:`asynchronous generator`, are also supported. + +.. note:: When a callback *returns* an object, Scrapy iterates that object if + it supports iteration, except for :class:`dict`, :class:`~scrapy.Item`, + :class:`str` and :class:`bytes` objects, which are always handled as single + items. + +.. note:: In a generator callback, a ``return`` statement with a value does not + produce any output, since such a value is not part of what the generator + yields. Scrapy logs a warning when it detects such a callback, see + :setting:`WARN_ON_GENERATOR_RETURN_VALUE`. + +Before Scrapy acts on the output of a callback, that output goes through the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method +of your :ref:`spider middlewares `, which may modify +it or drop part of it. + +If a callback raises an exception, the :attr:`~scrapy.Request.errback` of the +request is *not* called. The exception goes through the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception` +method of your spider middlewares instead and, unless one of them handles it, +Scrapy logs it and sends the :signal:`spider_error` signal. + + +.. _callback-data: +.. _topics-request-response-ref-request-callback-arguments: + +Passing additional data to callback functions +--------------------------------------------- + +In some cases you may be interested in passing data to a callback in addition +to the response, e.g. data extracted from the response that triggered the +request. The following example shows how to achieve this by using the +:attr:`.Request.cb_kwargs` attribute: + +.. code-block:: python + + from scrapy import Request + + + def parse(self, response): + request = Request( + "http://www.example.com/index.html", + callback=self.parse_page2, + cb_kwargs=dict(main_url=response.url), + ) + request.cb_kwargs["foo"] = "bar" # add more arguments for the callback + yield request + + + def parse_page2(self, response, main_url, foo): + yield dict( + main_url=main_url, + other_url=response.url, + foo=foo, + ) + +:attr:`.Request.cb_kwargs` is the recommended way to pass your own data to a +callback. Use :attr:`.Request.meta` only for data aimed at :ref:`components +`, such as middlewares and extensions. + +.. _errbacks: +.. _topics-request-response-ref-errbacks: + +Errbacks +======== + +The errback of a request is a function that will be called when an exception +is raise while processing it. + +It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can +be used to track connection establishment timeouts, DNS errors etc. + +Here's an example spider logging all errors and catching some specific +errors if needed: + +.. code-block:: python + + from scrapy import Request, Spider + from scrapy.spidermiddlewares.httperror import HttpError + from twisted.internet.error import DNSLookupError + from twisted.internet.error import TimeoutError, TCPTimedOutError + + + class ErrbackSpider(Spider): + name = "errback_example" + start_urls = [ + "http://www.httpbin.org/", # HTTP 200 expected + "http://www.httpbin.org/status/404", # Not found error + "http://www.httpbin.org/status/500", # server issue + "http://www.httpbin.org:12345/", # non-responding host, timeout expected + "https://example.invalid/", # DNS error expected + ] + + async def start(self): + for u in self.start_urls: + yield Request( + u, + callback=self.parse_httpbin, + errback=self.errback_httpbin, + dont_filter=True, + ) + + def parse_httpbin(self, response): + self.logger.info(f"Got successful response from {response.url}") + # do something useful here... + + def errback_httpbin(self, failure): + # log all failures + self.logger.error(repr(failure)) + + # in case you want to do something special for some errors, + # you may need the failure's type: + + if failure.check(HttpError): + # these exceptions come from HttpError spider middleware + # you can get the non-200 response + response = failure.value.response + self.logger.error("HttpError on %s", response.url) + + elif failure.check(DNSLookupError): + # this is the original request + request = failure.request + self.logger.error("DNSLookupError on %s", request.url) + + elif failure.check(TimeoutError, TCPTimedOutError): + request = failure.request + self.logger.error("TimeoutError on %s", request.url) + + +.. _errback-cb_kwargs: + +Accessing additional data in errback functions +---------------------------------------------- + +In case of a failure to process the request, you may be interested in +accessing arguments to the callback functions so you can process further +based on the arguments in the errback. The following example shows how to +achieve this by using ``Failure.request.cb_kwargs``: + +.. code-block:: python + + from scrapy import Request + + + def parse(self, response): + request = Request( + "http://www.example.com/index.html", + callback=self.parse_page2, + errback=self.errback_page2, + cb_kwargs=dict(main_url=response.url), + ) + yield request + + + def parse_page2(self, response, main_url): + pass + + + def errback_page2(self, failure): + yield dict( + main_url=failure.request.cb_kwargs["main_url"], + ) + + .. _topics-request-meta: Request.meta special keys diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 69ff08fa1..8fbf0c52d 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -4,43 +4,31 @@ Spiders ======= -Spiders are classes which define how a certain site (or a group of sites) will be -scraped, including how to perform the crawl (i.e. follow links) and how to -extract structured data from their pages (i.e. scraping items). In other words, -Spiders are the place where you define the custom behaviour for crawling and -parsing pages for a particular site (or, in some cases, a group of sites). +Spiders are classes that define how a site, or a group of sites, is scraped: +which requests to send, and how to parse their responses to extract data and to +send additional requests. -For spiders, the scraping cycle goes through something like this: +A crawl goes as follows: -1. You start by generating the initial requests to crawl the first URLs, and - specify a callback function to be called with the response downloaded from - those requests. +1. Scrapy iterates the :meth:`~scrapy.Spider.start` method of the spider to + get the initial requests. By default, that method yields a + :class:`~scrapy.Request` object for each URL in + :attr:`~scrapy.Spider.start_urls`, with :meth:`~scrapy.Spider.parse` as + :ref:`callback `. - The first requests to perform are obtained by iterating the - :meth:`~scrapy.Spider.start` method, which by default yields a - :class:`~scrapy.Request` object for each URL in the - :attr:`~scrapy.Spider.start_urls` spider attribute, with the - :attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback` - function to handle each :class:`~scrapy.http.Response`. +2. Scrapy downloads each request and calls its callback with the resulting + :class:`~scrapy.http.Response`. -2. In the callback function, you parse the response (web page) and return - :ref:`item objects `, - :class:`~scrapy.Request` objects, or an iterable of these objects. - Those Requests will also contain a callback (maybe - the same) and will then be downloaded by Scrapy and then their - response handled by the specified callback. +3. Callbacks parse the response, typically using :ref:`topics-selectors`, and + return or yield :ref:`item objects ` with the extracted data + and :class:`~scrapy.Request` objects to continue the crawl, which go back + to step 2. See :ref:`callback-output`. -3. In callback functions, you parse the page contents, typically using - :ref:`topics-selectors` (but you can also use BeautifulSoup, lxml or whatever - mechanism you prefer) and generate items with the parsed data. +4. Items go through :ref:`item pipelines `, and are + usually stored through :ref:`topics-feed-exports`. -4. Finally, the items returned from the spider will be typically persisted to a - database (in some :ref:`Item Pipeline `) or written to - a file using :ref:`topics-feed-exports`. - -Even though this cycle applies (more or less) to any kind of spider, there are -different kinds of default spiders bundled into Scrapy for different purposes. -We will talk about those types here. +Scrapy includes different spider classes for different purposes, described +below. .. _topics-spiders-ref: @@ -191,22 +179,7 @@ scrapy.Spider .. automethod:: start - .. method:: parse(response) - - This is the default callback used by Scrapy to process downloaded - responses, when their requests don't specify a callback. - - The ``parse`` method is in charge of processing the response and returning - scraped data and/or more URLs to follow. Other Requests callbacks have - the same requirements as the :class:`~scrapy.Spider` class. - - This method, as well as any other Request callback, must return a - :class:`~scrapy.Request` object, an :ref:`item object `, an - iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects - `, or ``None``. - - :param response: the response to parse - :type response: :class:`~scrapy.http.Response` + .. automethod:: parse .. method:: closed(reason) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 68847283a..7d67bb6d7 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -169,7 +169,8 @@ class Request(object_ref): #: #: The callable must expect the response as its first parameter, and #: support any additional keyword arguments set through - #: :attr:`cb_kwargs`. + #: :attr:`cb_kwargs`. See :ref:`writing-callbacks` and + #: :ref:`callback-output`. #: #: In addition to an arbitrary callable, the following values are also #: supported: @@ -190,8 +191,7 @@ class Request(object_ref): #: raises exceptions for non-2xx responses by default, sending them #: to the :attr:`errback` instead. #: - #: .. seealso:: - #: :ref:`topics-request-response-ref-request-callback-arguments` + #: .. seealso:: :ref:`callbacks` self.callback: CallbackT | None = callback #: :class:`~collections.abc.Callable` to handle exceptions raised @@ -200,7 +200,7 @@ class Request(object_ref): #: The callable must expect a :exc:`~twisted.python.failure.Failure` as #: its first parameter. #: - #: .. seealso:: :ref:`topics-request-response-ref-errbacks` + #: .. seealso:: :ref:`errbacks` self.errback: Callable[[Failure], Any] | None = errback self._cookies: CookiesT | None = cookies or None diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 02dfa2ac6..6244e3264 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -143,6 +143,22 @@ class Spider(object_ref): else: def parse(self, response: Response, **kwargs: Any) -> Any: + """Process *response*, i.e. extract data from it and generate new + requests. + + This is the default :ref:`callback `: Scrapy uses + it for the response to any request that does not define a + :attr:`~scrapy.Request.callback`, such as the requests that + :meth:`start` yields by default. + + Any :attr:`~scrapy.Request.cb_kwargs` of the request are passed as + keyword parameters. + + Spiders must define this method, unless every request that they + send defines a callback. + + See :ref:`callback-output` about the supported return values. + """ raise NotImplementedError( f"{self.__class__.__name__}.parse callback is not defined" ) From 3fc7148c5ec7537c000af27b705e33d797429a3a Mon Sep 17 00:00:00 2001 From: Janit Rajkarnikar <108281535+aniJani@users.noreply.github.com> Date: Thu, 30 Jul 2026 10:18:45 -0500 Subject: [PATCH 026/111] Don't evaluate annotations when inspecting signatures (#7818) Since Python 3.14 (PEP 649) annotations are evaluated lazily, so inspect.signature() raises NameError for callables whose annotations reference names imported only under TYPE_CHECKING. This broke middleware registration (via argument_is_required()) and custom stats collectors (via _warn_spider_arg) for user code with such annotations. Use annotation_format=Format.FORWARDREF on 3.14+: parameter names, kinds and defaults are unchanged, and unresolvable annotations become ForwardRef proxies instead of raising. Resolves #7796. --- scrapy/utils/decorators.py | 3 ++- scrapy/utils/python.py | 21 ++++++++++++++++++++- tests/test_utils_decorators.py | 30 ++++++++++++++++++++++++++++-- tests/test_utils_python.py | 21 ++++++++++++++++++++- 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 4960dc27a..2924c81f9 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -10,6 +10,7 @@ from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncio import run_in_thread from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.python import _signature if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable, Coroutine @@ -109,7 +110,7 @@ def _warn_spider_arg( ): """Decorator to warn if a ``spider`` argument is passed to a function.""" - sig = inspect.signature(func) + sig = _signature(func) def check_args(*args: _P.args, **kwargs: _P.kwargs) -> None: bound = sig.bind(*args, **kwargs) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 8a7517c1d..40fc05257 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -178,11 +178,30 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) +# PEP 649 (Python 3.14+) made annotation evaluation lazy, so inspect.signature() +# can raise NameError for names imported only under TYPE_CHECKING. We only need +# parameter names, kinds and defaults, so leave such annotations as ForwardRefs. +if sys.version_info >= (3, 14): + from annotationlib import Format + + def _signature(func: Callable[..., Any]) -> inspect.Signature: + return inspect.signature(func, annotation_format=Format.FORWARDREF) + +else: + + def _signature(func: Callable[..., Any]) -> inspect.Signature: + return inspect.signature(func) + + def get_func_args_dict( func: Callable[..., Any], stripself: bool = False ) -> Mapping[str, inspect.Parameter]: """Return the argument dict of a callable object. + Annotations are not evaluated, so on Python 3.14 and later the ``annotation`` + attribute of the returned parameters may be a ``ForwardRef`` instead of the + resolved type. + .. versionadded:: 2.14 """ if not callable(func): @@ -190,7 +209,7 @@ def get_func_args_dict( args: Mapping[str, inspect.Parameter] try: - sig = inspect.signature(func) + sig = _signature(func) except ValueError: return {} diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.py index 807294a57..4c29d2917 100644 --- a/tests/test_utils_decorators.py +++ b/tests/test_utils_decorators.py @@ -1,7 +1,8 @@ from __future__ import annotations +import sys import warnings -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import pytest from twisted.internet.defer import Deferred @@ -12,7 +13,7 @@ from scrapy.utils.defer import maybe_deferred_to_future from tests.utils.decorators import coroutine_test if TYPE_CHECKING: - from collections.abc import AsyncGenerator + from collections.abc import AsyncGenerator, Callable class TestDeprecated: @@ -77,6 +78,31 @@ class TestWarnSpiderArg: ): assert parse("response", spider="spider") == "response" + @pytest.mark.skipif( + sys.version_info < (3, 14), + reason="annotations are only lazily evaluated since Python 3.14 (PEP 649)", + ) + def test_sync_warns_with_unresolvable_annotations(self): + # dont_inherit=True, or the module's future import stringizes the annotations + namespace: dict[str, Any] = {} + exec( # pylint: disable=exec-used + compile( + "def parse(response: OnlyAtTypeCheckingTime," + " spider: OnlyAtTypeCheckingTime | None = None): return response", + "", + "exec", + dont_inherit=True, + ), + namespace, + ) + parse_func: Callable[..., str] = namespace["parse"] + parse = _warn_spider_arg(parse_func) + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Passing a 'spider' argument" + ): + assert parse("response", spider="spider") == "response" + def test_sync_no_warning_without_spider_arg(self): @_warn_spider_arg def parse(response: str, spider: str | None = None) -> str: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index c3b5dfc99..099b5ccc2 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -4,7 +4,7 @@ import functools import operator import platform import sys -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar import pytest @@ -190,6 +190,25 @@ def test_get_func_args(): ] +@pytest.mark.skipif( + sys.version_info < (3, 14), + reason="annotations are only lazily evaluated since Python 3.14 (PEP 649)", +) +def test_get_func_args_unresolvable_annotations(): + # dont_inherit=True, or the module's future import stringizes the annotations + namespace: dict[str, Any] = {} + exec( # pylint: disable=exec-used + compile( + "def f(a: OnlyAtTypeCheckingTime, b: int = 1) -> OnlyAtTypeCheckingTime: pass", + "", + "exec", + dont_inherit=True, + ), + namespace, + ) + assert get_func_args(namespace["f"]) == ["a", "b"] + + @pytest.mark.parametrize( ("value", "expected"), [ From 7436afc95f521482b1b1473c88f6b5ab1d430bea Mon Sep 17 00:00:00 2001 From: Adrian Date: Thu, 30 Jul 2026 17:27:27 +0200 Subject: [PATCH 027/111] Improve test coverage for scrapy.extensions (#7809) --- scrapy/extensions/closespider.py | 2 +- scrapy/extensions/feedexport.py | 4 +- scrapy/extensions/memusage.py | 2 +- scrapy/extensions/periodic_log.py | 3 +- tests/test_downloadermiddleware_httpcache.py | 71 ++++++++++++++++++++ tests/test_extension_memusage.py | 66 +++++++++++++++++- tests/test_extension_periodic_log.py | 36 ++++++++++ tests/test_extension_telnet.py | 55 ++++++++++++++- tests/test_feedexport.py | 26 +++++++ tests/test_feedexport_postprocess.py | 10 +++ tests/test_feedexport_storages.py | 15 +++++ 11 files changed, 278 insertions(+), 12 deletions(-) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index a4362b182..9cb792e30 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -119,7 +119,7 @@ class CloseSpider: self.task = None if self.task_no_item: - if self.task_no_item.running: + if self.task_no_item.running: # pragma: no branch self.task_no_item.stop() self.task_no_item = None diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 448279546..118462bc9 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -465,7 +465,7 @@ class FeedSlot: ) def finish_exporting(self) -> None: - if self._exporting: + if self._exporting: # pragma: no branch assert self.exporter self.exporter.finish_exporting() self._exporting = False @@ -553,7 +553,7 @@ class FeedExporter: for slot in self.slots: self._schedule_slot_close(slot, spider) - if self._pending_close_tasks: + if self._pending_close_tasks: # pragma: no branch if is_asyncio_available(): await asyncio.wait( cast("list[asyncio.Task[None]]", list(self._pending_close_tasks)) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 1444c8941..e0e289ce8 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -94,7 +94,7 @@ class MemoryUsage: def engine_stopped(self) -> None: for tsk in self.tasks: - if tsk.running: + if tsk.running: # pragma: no branch tsk.stop() def update(self) -> None: diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index cd35c8165..cbcc8b70e 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -38,7 +38,6 @@ class PeriodicLog: ): self.stats: StatsCollector = stats self.interval: float = interval - self.multiplier: float = 60.0 / self.interval self.task: AsyncioLoopingCall | LoopingCall | None = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) @@ -165,5 +164,5 @@ class PeriodicLog: def spider_closed(self, spider: Spider, reason: str) -> None: self.log() - if self.task and self.task.running: + if self.task and self.task.running: # pragma: no branch self.task.stop() diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index ce56ee11d..6e8486eb8 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -14,6 +14,7 @@ import pytest from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.exceptions import IgnoreRequest +from scrapy.extensions.httpcache import DummyPolicy from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.test import get_crawler @@ -24,6 +25,14 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler +class AlwaysStalePolicy(DummyPolicy): + """:class:`~scrapy.extensions.httpcache.DummyPolicy` that always + revalidates cached responses.""" + + def is_cached_response_fresh(self, cachedresponse, request): + return False + + class TestBase: """Base class with common setup and helper methods.""" @@ -282,6 +291,21 @@ class DummyPolicyTestMixin(PolicyTestMixin): self.assertEqualResponse(self.response, response) assert "cached" in response.flags + def test_revalidation_keeps_cached_response(self): + # The dummy policy considers every cached response valid, so a policy + # that subclasses it to force revalidation always gets the cached + # response back, whatever the new response is. + with self._middleware(HTTPCACHE_POLICY=AlwaysStalePolicy) as mw: + assert mw.process_request(self.request) is None + mw.process_response(self.request, self.response) + + assert mw.process_request(self.request) is None + fresh_response = self.response.replace(body=b"new body") + response = mw.process_response(self.request, fresh_response) + self.assertEqualResponse(self.response, response) + assert "cached" in response.flags + assert mw.stats.get_value("httpcache/revalidate") == 1 + class RFC2616PolicyTestMixin(PolicyTestMixin): """Mixin containing RFC2616 policy specific test methods.""" @@ -553,6 +577,53 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): else: assert "cached" in res5.flags + def test_middleware_ignore_schemes(self): + # file responses are not cached by default + req = Request("file:///tmp/t.txt") + res = Response(req.url, headers={"Expires": self.tomorrow}) + with self._middleware() as mw: + assert mw.process_request(req) is None + mw.process_response(req, res) + + assert mw.storage.retrieve_response(mw.crawler.spider, req) is None + assert mw.process_request(req) is None + + def test_max_stale_with_value(self): + # A response that expired one day ago. + headers = {"Date": self.yesterday, "Expires": self.yesterday} + with self._middleware() as mw: + req0 = Request("http://example.com") + res0 = Response(req0.url, headers=headers) + self._process_requestresponse(mw, req0, res0) + + # max-stale greater than the staleness of the cached response + req1 = req0.replace(headers={"Cache-Control": "max-stale=172800"}) + res1 = mw.process_request(req1) + assert isinstance(res1, Response) + assert "cached" in res1.flags + + # max-stale lower than the staleness of the cached response + req2 = req0.replace(headers={"Cache-Control": "max-stale=60"}) + assert mw.process_request(req2) is None + + # a non-integer max-stale value is ignored + req3 = req0.replace(headers={"Cache-Control": "max-stale=soon"}) + assert mw.process_request(req3) is None + + def test_response_dated_in_the_future(self): + # A Date header ahead of the local clock must not make the cached + # response look aged. + headers = {"Date": self.tomorrow, "Cache-Control": "max-age=10"} + with self._middleware() as mw: + req0 = Request("http://example.com") + res0 = Response(req0.url, headers=headers) + res1 = self._process_requestresponse(mw, req0, res0) + assert "cached" not in res1.flags + + res2 = self._process_requestresponse(mw, req0, None) + self.assertEqualResponse(res1, res2) + assert "cached" in res2.flags + def test_process_exception(self): with self._middleware() as mw: res0 = Response(self.request.url, headers={"Expires": self.yesterday}) diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py index a474725d8..76e8ca5d6 100644 --- a/tests/test_extension_memusage.py +++ b/tests/test_extension_memusage.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging import sys +from typing import TYPE_CHECKING import pytest @@ -13,8 +14,12 @@ from scrapy.extensions.memusage import MemoryUsage from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.utils import OneShotLoop +from tests.utils.cmdline import proc from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from tests.mockserver.http import MockServer + # MemoryUsage relies on the stdlib 'resource' module (not available on Windows) pytestmark = pytest.mark.skipif( sys.platform.startswith("win"), @@ -25,6 +30,14 @@ pytestmark = pytest.mark.skipif( MB = 1024 * 1024 +class TwoShotLoop(OneShotLoop): + """Like :class:`OneShotLoop`, but runs the check twice.""" + + def start(self, interval: float, now: bool = True) -> None: + super().start(interval, now=now) + self.func() + + class _LoopSpider(Spider): name = "loop-data-spider" @@ -50,6 +63,49 @@ def test_memusage_disabled() -> None: MemoryUsage.from_crawler(get_crawler(settings_dict=settings)) +def test_memusage_limit_stops_crawler_without_spider(mockserver: MockServer) -> None: + # The Scrapy shell starts the engine without opening a spider, so the + # whole crawler is stopped instead of a spider being closed. + _, out, err = proc( + "shell", + mockserver.url("/text"), + "-c", + "response.status", + "--set", + "MEMUSAGE_LIMIT_MB=1", + ) + assert "Memory usage exceeded 1MiB" in err + assert "200" in out + + +@coroutine_test +async def test_memusage_below_thresholds_logs_peak( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = { + "MEMUSAGE_LIMIT_MB": 100, + "MEMUSAGE_WARNING_MB": 50, + "MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01, + "TELNETCONSOLE_ENABLED": False, + "LOG_LEVEL": "INFO", + } + + monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda _: 25 * MB) + + crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) + + with caplog.at_level(logging.INFO, logger="scrapy.extensions.memusage"): + await crawler.crawl_async(url="data:,", loops=1) + + assert crawler.stats + assert crawler.stats.get_value("memusage/limit_reached") is None + assert crawler.stats.get_value("memusage/warning_reached") is None + assert crawler.stats.get_value("memusage/max") == 25 * MB + assert crawler.stats.get_value("finish_reason") == "finished" + assert any("Peak memory usage is 25MiB" in r.getMessage() for r in caplog.records) + + @coroutine_test async def test_memusage_limit_closes_spider_with_reason_and_error_log( caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch @@ -92,8 +148,9 @@ async def test_memusage_warning_logs_but_allows_normal_finish( "LOG_LEVEL": "INFO", } - # Avoid background LoopingCall that can log after the test finishes. - monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + # Avoid background LoopingCall that can log after the test finishes; check + # twice, since the warning is only meant to be reported once. + monkeypatch.setattr(memusage_mod, "create_looping_call", TwoShotLoop) monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB) crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) @@ -112,4 +169,7 @@ async def test_memusage_warning_logs_but_allows_normal_finish( assert crawler.stats assert crawler.stats.get_value("memusage/warning_reached") == 1 assert crawler.stats.get_value("finish_reason") == "finished" - assert any("memory usage reached" in r.getMessage().lower() for r in caplog.records) + warnings_logged = [ + r for r in caplog.records if "memory usage reached" in r.getMessage().lower() + ] + assert len(warnings_logged) == 1 diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index ffe7a0dc7..723fb6e17 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,8 +1,13 @@ from __future__ import annotations import datetime +import json +import logging from typing import TYPE_CHECKING, Any +import pytest + +from scrapy.exceptions import NotConfigured from scrapy.extensions.periodic_log import PeriodicLog from scrapy.utils.test import get_crawler @@ -86,6 +91,14 @@ class TestPeriodicLog: assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) + def test_no_interval(self): + with pytest.raises(NotConfigured): + extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 0}) + + def test_nothing_enabled(self): + with pytest.raises(NotConfigured): + extension({"LOGSTATS_INTERVAL": 60}) + @coroutine_test async def test_log_delta(self): def emulate( @@ -212,3 +225,26 @@ class TestPeriodicLog: {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}}, lambda k, v: "downloader/" in k and "bytes" not in k, ) + + @coroutine_test + async def test_log_timing(self, caplog: pytest.LogCaptureFixture) -> None: + settings = { + "EXTENSIONS": {"scrapy.extensions.periodic_log.PeriodicLog": 0}, + "PERIODIC_LOG_TIMING_ENABLED": True, + "LOGSTATS_INTERVAL": 30, + } + crawler = get_crawler(MetaSpider, settings) + with caplog.at_level(logging.INFO, logger="scrapy.extensions.periodic_log"): + await crawler.crawl_async() + + records = [ + r for r in caplog.records if r.name == "scrapy.extensions.periodic_log" + ] + assert records, "PeriodicLog logged nothing" + # Only the timing section is enabled, and it is logged on spider close. + data = json.loads(records[-1].getMessage()) + assert list(data) == ["time"] + assert data["time"]["log_interval"] == 30 + assert data["time"]["log_interval_real"] >= 0 + assert data["time"]["elapsed"] >= 0 + assert data["time"]["start_time"] <= data["time"]["utcnow"] diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 20c801558..fca0e3153 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -7,7 +7,8 @@ import pytest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials -from scrapy.extensions.telnet import TelnetConsole +from scrapy import Spider +from scrapy.extensions.telnet import TelnetConsole, update_telnet_vars from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.utils.decorators import coroutine_test @@ -16,16 +17,20 @@ if TYPE_CHECKING: from collections.abc import Generator from scrapy.crawler import Crawler + from scrapy.http import Response pytestmark = pytest.mark.requires_reactor # TelnetConsole requires a reactor -def _get_crawler(settings_dict: dict[str, Any] | None = None) -> Crawler: +def _get_crawler( + spidercls: type[Spider] | None = None, + settings_dict: dict[str, Any] | None = None, +) -> Crawler: settings = { "TELNETCONSOLE_ENABLED": True, **(settings_dict or {}), } - return get_crawler(settings_dict=settings) + return get_crawler(spidercls, settings_dict=settings) @contextmanager @@ -84,3 +89,47 @@ def test_invalid_reversed_portrange() -> None: console = TelnetConsole(_get_crawler(settings_dict=settings)) with pytest.raises(ValueError, match=r"invalid portrange: \[2, 1\]"): console.start_listening() + + +@coroutine_test +async def test_telnet_vars() -> None: + """Log into the console of a running crawl, which is when the telnet + variables are built.""" + received: list[dict[str, Any]] = [] + + def on_update_telnet_vars(telnet_vars: dict[str, Any]) -> None: + received.append(telnet_vars) + + class TelnetSpider(Spider): + name = "telnet" + start_urls = ["data:,"] + + async def parse(self, response: Response) -> None: + assert self.crawler.extensions + console = next( + ext + for ext in self.crawler.extensions.middlewares + if isinstance(ext, TelnetConsole) + ) + creds = credentials.UsernamePassword( + console.username.encode("utf8"), console.password.encode("utf8") + ) + portal = console.protocol().protocolArgs[0] + await maybe_deferred_to_future(portal.login(creds, None, ITelnetProtocol)) + + crawler = _get_crawler(TelnetSpider) + crawler.signals.connect(on_update_telnet_vars, signal=update_telnet_vars) + await crawler.crawl_async() + + assert len(received) == 1 + telnet_vars = received[0] + assert telnet_vars["crawler"] is crawler + assert telnet_vars["engine"] is crawler.engine + assert telnet_vars["spider"] is crawler.spider + assert telnet_vars["extensions"] is crawler.extensions + assert telnet_vars["stats"] is crawler.stats + assert telnet_vars["settings"] is crawler.settings + assert callable(telnet_vars["est"]) + assert callable(telnet_vars["p"]) + assert callable(telnet_vars["prefs"]) + assert "telnetconsole.html" in telnet_vars["help"] diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1cca287af..40a763efd 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -24,6 +24,7 @@ from scrapy.extensions.feedexport import ( FeedExporter, FeedSlot, FileFeedStorage, + ItemFilter, apply_uri_params, ) from scrapy.utils.python import to_unicode @@ -1289,6 +1290,13 @@ class TestFeedExporterSignals: assert self.feed_exporter_closed_received +class TestItemFilter: + def test_no_feed_options(self): + item_filter = ItemFilter(None) + assert item_filter.item_classes == () + assert item_filter.accepts(MyItem({"foo": "bar"})) + + class TestFeedExportInit: def test_unsupported_storage(self): settings = { @@ -1300,6 +1308,24 @@ class TestFeedExportInit: with pytest.raises(NotConfigured): FeedExporter.from_crawler(crawler) + def test_disabled_storage(self, caplog: pytest.LogCaptureFixture): + class DisabledFeedStorage: + def __init__(self, uri, *, feed_options=None): + raise NotConfigured("not today") + + settings = { + "FEED_STORAGES": {"disabled": DisabledFeedStorage}, + "FEEDS": { + "disabled://uri": {}, + }, + } + crawler = get_crawler(settings_dict=settings) + with caplog.at_level(logging.ERROR), pytest.raises(NotConfigured): + FeedExporter.from_crawler(crawler) + assert ( + "Disabled feed storage scheme: disabled. Reason: not today" in caplog.text + ) + def test_unsupported_format(self): settings = { "FEEDS": { diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py index f120ce36f..36d8586ce 100644 --- a/tests/test_feedexport_postprocess.py +++ b/tests/test_feedexport_postprocess.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any import pytest +from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.test import get_crawler from tests.utils.bases.feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test @@ -87,6 +88,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data_stream.seek(0) return data_stream.read() + def test_tell_reports_target_file_position(self): + """Exporters that wrap the file they get, e.g. through + :class:`io.TextIOWrapper`, need it to report a position.""" + file = BytesIO() + manager = PostProcessingManager([self.MyPlugin1], file, {}) + assert manager.tell() == 0 + manager.write(b"foo") + assert manager.tell() == file.tell() == 3 + @coroutine_test async def test_gzip_plugin(self): filename = self._named_tempfile("gzip_file") diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index b1e3787fd..4d28872b7 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging import os import string +import sys import tempfile from io import BytesIO from pathlib import Path @@ -14,6 +15,7 @@ import pytest from w3lib.url import path_to_file_uri import scrapy +from scrapy.exceptions import NotConfigured from scrapy.extensions.feedexport import ( BlockingFeedStorage, FileFeedStorage, @@ -166,6 +168,12 @@ class TestFTPFeedStorage: st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) assert st.password == string.punctuation + def test_uri_without_hostname(self): + with pytest.raises( + ValueError, match="Got a storage URI without a hostname: ftp:///some_path" + ): + FTPFeedStorage("ftp:///some_path") + class MyBlockingFeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: @@ -205,6 +213,13 @@ class TestBlockingFeedStorage: b.open(spider=spider) +def test_s3_without_boto3(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "boto3", None) + monkeypatch.setitem(sys.modules, "boto3.session", None) + with pytest.raises(NotConfigured, match="missing boto3 library"): + S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") + + @pytest.mark.requires_boto3 class TestS3FeedStorage: def test_parse_credentials(self): From 259be5d2dd02ab1be876bf4b6be1849cbab88c6f Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 08:20:21 +0200 Subject: [PATCH 028/111] CI: Run coverage in as few jobs as necessary (#7834) * CI: Run coverage in as few jobs as necessary * Complete test coverage --- .github/workflows/tests-macos.yml | 9 +++++++-- .github/workflows/tests-ubuntu.yml | 14 +++++++++++++- .github/workflows/tests-windows.yml | 4 +++- tests/test_utils_console.py | 19 ++++++++++++++++++- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 0409b3ef2..566b34e50 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -14,14 +14,18 @@ jobs: tests: runs-on: macos-latest env: - PYTEST_ADDOPTS: -n auto + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13"] env: - TOXENV: py include: + - python-version: '3.14' + env: + TOXENV: py + coverage: true - python-version: '3.14' env: TOXENV: no-reactor @@ -41,6 +45,7 @@ jobs: tox - name: Upload coverage report + if: ${{ matrix.coverage }} uses: codecov/codecov-action@v5 - name: Upload test results diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 15f25d9b8..ad2bcfce7 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -14,7 +14,7 @@ jobs: tests: runs-on: ubuntu-latest env: - PYTEST_ADDOPTS: -n auto + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} strategy: fail-fast: false matrix: @@ -34,12 +34,15 @@ jobs: - python-version: "3.14" env: TOXENV: py + coverage: true - python-version: "3.14" env: TOXENV: default-reactor + coverage: true - python-version: "3.14" env: TOXENV: no-reactor + coverage: true # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: @@ -49,12 +52,15 @@ jobs: - python-version: "3.10.19" env: TOXENV: min + coverage: true - python-version: "3.10.19" env: TOXENV: min-default-reactor + coverage: true - python-version: "3.10.19" env: TOXENV: min-no-reactor + coverage: true # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: @@ -62,16 +68,20 @@ jobs: - python-version: "3.10.19" env: TOXENV: min-extra-deps + coverage: true - python-version: "3.10.19" env: TOXENV: min-botocore + coverage: true - python-version: "3.14" env: TOXENV: extra-deps + coverage: true - python-version: "3.14" env: TOXENV: no-reactor-extra-deps + coverage: true # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: @@ -79,6 +89,7 @@ jobs: - python-version: "3.14" env: TOXENV: botocore + coverage: true steps: - uses: actions/checkout@v6 @@ -104,6 +115,7 @@ jobs: tox - name: Upload coverage report + if: ${{ matrix.coverage }} uses: codecov/codecov-action@v5 - name: Upload test results diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index f413782bc..c33f96b12 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -14,7 +14,7 @@ jobs: tests: runs-on: windows-latest env: - PYTEST_ADDOPTS: -n auto + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} strategy: fail-fast: false matrix: @@ -34,6 +34,7 @@ jobs: - python-version: "3.14" env: TOXENV: py + coverage: true - python-version: "3.14" env: TOXENV: default-reactor @@ -68,6 +69,7 @@ jobs: tox - name: Upload coverage report + if: ${{ matrix.coverage }} uses: codecov/codecov-action@v5 - name: Upload test results diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index ad9aa3dff..ab0c72d8a 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -4,7 +4,7 @@ from importlib.util import find_spec import pytest -from scrapy.utils.console import get_shell_embed_func +from scrapy.utils.console import get_shell_embed_func, start_python_console def test_get_shell_embed_func(): @@ -59,3 +59,20 @@ def test_get_shell_embed_func_default(): else: expected = "_embed_standard_shell" assert shell.__name__ == expected + + +def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None: + def embed(namespace: dict[str, object], banner: str) -> None: + raise SystemExit + + monkeypatch.setattr( + "scrapy.utils.console.get_shell_embed_func", lambda shells: embed + ) + start_python_console() + + +def test_start_python_console_no_shell(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "scrapy.utils.console.get_shell_embed_func", lambda shells: None + ) + start_python_console() From 434fd1154ad26e04ea2e438ab5d3b8dd70e73885 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 08:38:33 +0200 Subject: [PATCH 029/111] Improve pre-crawler setting docs (#7835) --- docs/topics/commands.rst | 2 ++ docs/topics/settings.rst | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index ee2c3a3cd..e8a843e80 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -665,6 +665,8 @@ Example: COMMANDS_MODULE = "mybot.commands" +.. note:: This is a :ref:`pre-crawler setting `. + .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html Register commands via setup.py entry points diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1b6851d04..81055afc2 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -305,10 +305,21 @@ These settings cannot be :ref:`set from a spider `. These settings are: -- :setting:`TWISTED_REACTOR_ENABLED` +- :setting:`ADDONS` +- :setting:`COMMANDS_MODULE` +- :setting:`FORCE_CRAWLER_PROCESS` - :setting:`SPIDER_LOADER_CLASS` and settings used by the corresponding spider loader class, e.g. :setting:`SPIDER_MODULES` and :setting:`SPIDER_LOADER_WARN_ONLY` for the default spider loader class. +- :setting:`TWISTED_REACTOR_ENABLED` + +:setting:`ADDONS` is a special case: it can be set from a spider, but the +``update_pre_crawler_settings()`` method of :ref:`add-ons ` +enabled that way is not called. + +:setting:`TWISTED_REACTOR` also acts as a pre-crawler setting when running a +:ref:`command that needs a CrawlerProcess `, +since its project-level value determines the crawler process class. .. _reactor-settings: @@ -409,6 +420,9 @@ Default: ``{}`` A dict containing paths to the add-ons enabled in your project and their priorities. For more information, see :ref:`topics-addons`. +.. note:: This is a :ref:`pre-crawler setting `, with a + caveat described in that section. + .. setting:: ASYNCIO_EVENT_LOOP ASYNCIO_EVENT_LOOP @@ -1402,6 +1416,8 @@ When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``, Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a non-default value in :ref:`per-spider settings `. +.. note:: This is a :ref:`pre-crawler setting `. + .. setting:: FTP_PASSIVE_MODE FTP_PASSIVE_MODE From 37661508dbaec28bd68e2d05306e20b7ee0f5652 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 08:59:52 +0200 Subject: [PATCH 030/111] Add RobotParser.crawl_delay() and a robots_parsed signal (#7830) * Add RobotParser.crawl_delay() and a robots_parsed signal * Improve test coverage --- docs/topics/signals.rst | 21 +++++++++++++++ scrapy/downloadermiddlewares/robotstxt.py | 12 +++++++-- scrapy/robotstxt.py | 21 +++++++++++++++ scrapy/signals.py | 1 + tests/test_downloadermiddleware_robotstxt.py | 17 ++++++++++++ tests/test_robotstxt_interface.py | 27 ++++++++++++++++++++ 6 files changed, 97 insertions(+), 2 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 03996bee6..f7f9f5cca 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -504,6 +504,27 @@ headers_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.Spider` object +robots_parsed +~~~~~~~~~~~~~ + +.. signal:: robots_parsed +.. function:: robots_parsed(robotparser, request) + + .. versionadded:: VERSION + + Sent by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it + downloads and parses a :file:`robots.txt` file, for the host that *request* + targets. + + This signal supports :ref:`asynchronous handlers `. + + :param robotparser: the parser holding the parsed :file:`robots.txt` contents + :type robotparser: :class:`~scrapy.robotstxt.RobotParser` object + + :param request: the request that triggered the :file:`robots.txt` download + :type request: :class:`~scrapy.Request` object + Response signals ---------------- diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7d0c17884..81a3a887f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred +from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK @@ -98,7 +99,7 @@ class RobotsTxtMiddleware: assert self.crawler.stats try: resp = await self.crawler.engine.download_async(robotsreq) - self._parse_robots(resp, netloc) + await self._parse_robots(resp, netloc, request) except Exception as e: if not isinstance(e, IgnoreRequest): logger.error( @@ -115,13 +116,20 @@ class RobotsTxtMiddleware: return await maybe_deferred_to_future(parser) return parser - def _parse_robots(self, response: Response, netloc: str) -> None: + async def _parse_robots( + self, response: Response, netloc: str, request: Request + ) -> None: assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/response_count") self.crawler.stats.inc_value( f"robotstxt/response_status_count/{response.status}" ) rp = self._parserimpl.from_crawler(self.crawler, response.body) + await self.crawler.signals.send_catch_log_async( + signal=signals.robots_parsed, + robotparser=rp, + request=request, + ) rp_dfd = self._parsers[netloc] assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = rp diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 0c64ea5a5..b54011784 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -67,6 +67,15 @@ class RobotParser(metaclass=ABCMeta): :type user_agent: str or bytes """ + def crawl_delay(self, user_agent: str | bytes) -> float | None: + """Return the ``Crawl-delay`` directive for ``user_agent`` as a number + of seconds, or ``None`` if it is not set or the backend does not support + it. + + .. versionadded:: VERSION + """ + return None + class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -85,6 +94,10 @@ class PythonRobotParser(RobotParser): url = to_unicode(url) return self.rp.can_fetch(user_agent, url) + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) + class RerpRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -105,6 +118,10 @@ class RerpRobotParser(RobotParser): url = to_unicode(url) return cast("bool", self.rp.is_allowed(user_agent, url)) + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.get_crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) + class ProtegoRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -121,3 +138,7 @@ class ProtegoRobotParser(RobotParser): user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) + + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) diff --git a/scrapy/signals.py b/scrapy/signals.py index 972f4fd60..3afeb6eab 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -21,6 +21,7 @@ response_received = object() response_downloaded = object() headers_received = object() bytes_received = object() +robots_parsed = object() item_scraped = object() item_dropped = object() item_error = object() diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index f82041a62..1f2575f8f 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -8,6 +8,7 @@ import pytest from twisted.internet.defer import Deferred, DeferredList from twisted.python import failure +from scrapy import signals from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse @@ -27,6 +28,7 @@ class TestRobotsTxtMiddleware: self.crawler: mock.MagicMock = mock.MagicMock() self.crawler.settings = Settings() self.crawler.engine.download_async = mock.AsyncMock() + self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[]) def teardown_method(self): del self.crawler @@ -74,6 +76,21 @@ Disallow: /some/randome/page.html Request("http://site.local/wiki/Käyttäjä:"), middleware ) + @coroutine_test + async def test_robotstxt_emits_robots_parsed_signal(self): + crawler = self._get_successful_crawler() + middleware = RobotsTxtMiddleware(crawler) + request = Request("http://site.local/allowed") + await self.assertNotIgnored(request, middleware) + calls = [ + kwargs + for _, kwargs in crawler.signals.send_catch_log_async.call_args_list + if kwargs.get("signal") is signals.robots_parsed + ] + assert len(calls) == 1 + assert calls[0]["request"] is request + assert calls[0]["robotparser"] is not None + @coroutine_test async def test_robotstxt_multiple_reqs(self) -> None: middleware = RobotsTxtMiddleware(self._get_successful_crawler()) diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index da94a4e95..ea67877f8 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -4,6 +4,7 @@ from scrapy.robotstxt import ( ProtegoRobotParser, PythonRobotParser, RerpRobotParser, + RobotParser, decode_robotstxt, ) from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER @@ -78,6 +79,16 @@ class BaseRobotParserTest: assert rp.allowed("https://site.local/index.html", "*") assert rp.allowed("https://site.local/disallowed", "*") + def test_crawl_delay(self): + robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n" + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body) + assert rp.crawl_delay("*") == 10.0 + + def test_crawl_delay_unset(self): + robotstxt_body = b"User-agent: *\nDisallow: /private\n" + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body) + assert rp.crawl_delay("*") is None + def test_unicode_url_and_useragent(self): robotstxt_robotstxt_body = """ User-Agent: * @@ -102,6 +113,22 @@ class BaseRobotParserTest: assert not rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt") +class TestRobotParser: + def test_crawl_delay_unsupported(self): + class AllowAllRobotParser(RobotParser): + @classmethod + def from_crawler(cls, crawler, robotstxt_body): + return cls() + + def allowed(self, url, user_agent): + return True + + rp = AllowAllRobotParser.from_crawler( + crawler=None, robotstxt_body=b"User-agent: *\nCrawl-delay: 10\n" + ) + assert rp.crawl_delay("*") is None + + class TestDecodeRobotsTxt: def test_native_string_conversion(self): robotstxt_body = b"User-agent: *\nDisallow: /\n" From 746bc7548d358ba96bfa74e9ce6ceb06f11c550d Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 09:17:34 +0200 Subject: [PATCH 031/111] Clarify crawl vs runspider in help and docs (#7832) --- docs/topics/commands.rst | 11 +++++++---- scrapy/commands/crawl.py | 2 +- scrapy/commands/runspider.py | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index e8a843e80..343193627 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -114,8 +114,8 @@ some usage help and the available commands:: scrapy [options] [args] Available commands: - crawl Run a spider fetch Fetch a URL using the Scrapy downloader + runspider Run a spider from a Python file, no project required [...] The first line will print the currently active project if you're inside a @@ -263,7 +263,9 @@ crawl * Syntax: ``scrapy crawl `` * Requires project: *yes* -Start crawling using a spider. +Start crawling using the spider with the given :attr:`~scrapy.Spider.name`, +which must be one of those that :command:`list` reports. To run a spider from a +file instead, use :command:`runspider`. Supported options: @@ -571,8 +573,9 @@ runspider * Syntax: ``scrapy runspider `` * Requires project: *no* -Run a spider self-contained in a Python file, without having to create a -project. +Run the spider defined in the given Python file, without requiring a project. + +Supported options: the same as :command:`crawl`. Example usage:: diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 866ba9f6b..4e086e057 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand): return "[options] " def short_desc(self) -> str: - return "Run a spider" + return "Run a spider of the current project, by name" def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) < 1: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 0b9036457..9cdb393ab 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -38,7 +38,7 @@ class Command(BaseRunSpiderCommand): return "[options] " def short_desc(self) -> str: - return "Run a self-contained spider (without creating a project)" + return "Run a spider from a Python file, no project required" def long_desc(self) -> str: return "Run the spider defined in the given file" From ea7c0af2f96400be88513172ea55579997e1b892 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 09:18:06 +0200 Subject: [PATCH 032/111] Use a single badge for all tests (#7836) --- README.rst | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 6235cb20c..651294add 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ :alt: Scrapy :width: 480px -|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki| +|version| |python_version| |tests| |coverage| |conda| |deepwiki| .. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg :target: https://pypi.org/pypi/Scrapy @@ -15,17 +15,9 @@ :target: https://pypi.org/pypi/Scrapy :alt: Supported Python Versions -.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu - :alt: Ubuntu - -.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - :alt: macOS - -.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows - :alt: Windows +.. |tests| image:: https://img.shields.io/github/check-runs/scrapy/scrapy/master?label=tests + :target: https://github.com/scrapy/scrapy/actions?query=branch%3Amaster + :alt: Tests .. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master From 3180116cd08942997548f62f90a66e01e64a690b Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 09:19:49 +0200 Subject: [PATCH 033/111] Improve test coverage for scrapy.commands (#7829) --- scrapy/commands/genspider.py | 5 +- scrapy/commands/parse.py | 2 +- tests/test_command_crawl.py | 12 +++ tests/test_command_fetch.py | 32 ++++++++ tests/test_command_genspider.py | 21 ++++- tests/test_command_parse.py | 125 ++++++++++++++++++++++++++++ tests/test_command_runspider.py | 6 ++ tests/test_command_shell.py | 28 +++++++ tests/test_commands.py | 139 ++++++++++++++++++++++++++++---- tests/utils/cmdline.py | 13 +++ 10 files changed, 363 insertions(+), 20 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 4277232c3..52f9cd4b0 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -32,10 +32,7 @@ def sanitize_module_name(module_name: str) -> str: def extract_domain(url: str) -> str: """Extract domain name from URL string""" - o = urlparse(url) - if o.scheme == "" and o.netloc == "": - o = urlparse("//" + url.lstrip("/")) - return o.netloc + return urlparse(url).netloc def verify_url_scheme(url: str) -> str: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 93194ded7..51caed57f 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -41,7 +41,7 @@ class Command(BaseRunSpiderCommand): spider: Spider | None = None items: ClassVar[dict[int, list[Any]]] = {} requests: ClassVar[dict[int, list[Request]]] = {} - spidercls: type[Spider] | None + spidercls: type[Spider] | None = None first_response = None diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py index 70c26e6d0..5306e3bf8 100644 --- a/tests/test_command_crawl.py +++ b/tests/test_command_crawl.py @@ -23,6 +23,18 @@ class TestCrawlCommand(TestProjectBase): _, _, stderr = self.crawl(code, proj_path, args=args) return stderr + def test_no_spider(self, proj_path: Path) -> None: + returncode, out, _ = proc("crawl", cwd=proj_path) + assert returncode == 2 + assert "Usage" in out + + def test_multiple_spiders(self, proj_path: Path) -> None: + returncode, _, err = proc("crawl", "myspider", "myspider2", cwd=proj_path) + assert returncode == 2 + assert ( + "running 'scrapy crawl' with more than one spider is not supported" in err + ) + def test_no_output(self, proj_path: Path) -> None: spider_code = """ import scrapy diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index d98dac968..c6a3afc91 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -2,13 +2,24 @@ from __future__ import annotations from typing import TYPE_CHECKING +import pytest + +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc if TYPE_CHECKING: + from pathlib import Path + from tests.mockserver.http import MockServer class TestFetchCommand: + @pytest.mark.parametrize("args", [(), ("not-a-url",), ("a:b", "c:d")]) + def test_bad_arguments(self, args: tuple[str, ...]) -> None: + returncode, out, _ = proc("fetch", *args) + assert returncode == 2 + assert "Usage" in out + def test_output(self, mockserver: MockServer) -> None: _, out, _ = proc("fetch", mockserver.url("/text")) assert out.strip() == "Works" @@ -36,3 +47,24 @@ class TestFetchCommand: "fetch", "-s", "TWISTED_REACTOR_ENABLED=False", mockserver.url("/text") ) assert out.strip() == "Works" + + +class TestFetchCommandWithSpider(TestProjectBase): + @pytest.fixture(autouse=True) + def create_files(self, proj_path: Path) -> None: + (proj_path / self.project_name / "spiders" / "myspider.py").write_text( + """ +import scrapy + +class MySpider(scrapy.Spider): + name = "myspider" + custom_settings = {"USER_AGENT": "myspider-user-agent"} +""", + encoding="utf-8", + ) + + def test_spider(self, proj_path: Path, mockserver: MockServer) -> None: + _, out, err = proc( + "fetch", "--spider", "myspider", mockserver.url("/echo"), cwd=proj_path + ) + assert "myspider-user-agent" in out, err diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py index 8bb6a2332..ddf25af4c 100644 --- a/tests/test_command_genspider.py +++ b/tests/test_command_genspider.py @@ -64,6 +64,24 @@ class TestGenspiderCommand(TestProjectBase): assert call("genspider", "--dump=basic", cwd=proj_path) == 0 assert call("genspider", "-d", "basic", cwd=proj_path) == 0 + @pytest.mark.parametrize( + "args", + [("--dump=nonexistent",), ("-t", "nonexistent", "test_name", "test.com")], + ) + def test_unknown_template(self, args: tuple[str, ...], proj_path: Path) -> None: + returncode, out, err = proc("genspider", *args, cwd=proj_path) + assert returncode == 0, err + assert "Unable to find template: nonexistent" in out + assert not (proj_path / self.project_name / "spiders" / "test_name.py").exists() + + def test_name_not_starting_with_a_letter(self, proj_path: Path) -> None: + """The module name, unlike the spider name, is prefixed with a letter.""" + _, out, err = proc("genspider", "1st_spider", "test.com", cwd=proj_path) + assert "Created spider '1st_spider'" in out, err + spider = proj_path / self.project_name / "spiders" / "a1st_spider.py" + assert spider.exists() + assert find_in_file(spider, r'name\s*=\s*"1st_spider"') is not None + @pytest.mark.skipif( sys.platform == "win32", reason="requires a POSIX shell editor script" ) @@ -87,7 +105,8 @@ class TestGenspiderCommand(TestProjectBase): ) def test_same_name_as_project(self, proj_path: Path) -> None: - assert call("genspider", self.project_name, cwd=proj_path) == 2 + _, out, err = proc("genspider", self.project_name, "test.com", cwd=proj_path) + assert "Cannot create a spider with the same name as your project" in out, err assert not ( proj_path / self.project_name / "spiders" / f"{self.project_name}.py" ).exists() diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 772cc82e2..e434055b7 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse import re from typing import TYPE_CHECKING +from urllib.parse import urlparse import pytest @@ -552,6 +553,130 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} content = '[\n{},\n{"foo": "bar"}\n]' assert file_path.read_text(encoding="utf-8") == content + @pytest.mark.parametrize("args", [(), ("not-a-url",), ("a:b", "c:d")]) + def test_bad_arguments(self, args: tuple[str, ...], proj_path: Path) -> None: + returncode, out, _ = proc("parse", *args, cwd=proj_path) + assert returncode == 2 + assert "Usage" in out + + @pytest.mark.parametrize( + ("option", "message"), + [ + ("--meta", "Invalid -m/--meta value"), + ("-m", "Invalid -m/--meta value"), + ("--cbkwargs", "Invalid --cbkwargs value"), + ], + ) + def test_invalid_json( + self, option: str, message: str, proj_path: Path, mockserver: MockServer + ) -> None: + returncode, _, err = proc( + "parse", + "--spider", + self.spider_name, + option, + "{invalid", + mockserver.url("/html"), + cwd=proj_path, + ) + assert returncode == 2 + assert message in err + + def test_unknown_spider(self, proj_path: Path, mockserver: MockServer) -> None: + returncode, _, err = proc( + "parse", + "--spider", + "nonexistent", + mockserver.url("/html"), + cwd=proj_path, + ) + assert returncode == 0, err + assert "Unable to find spider: nonexistent" in err + + def test_spider_found_by_url(self, proj_path: Path, mockserver: MockServer) -> None: + """Without --spider, the spider is chosen based on the URL.""" + url = mockserver.url("/html") + # The spider name doubles as a domain of the spider, and it is matched + # against the netloc of the URL, hence the port. + (proj_path / self.project_name / "spiders" / "urlspider.py").write_text( + f""" +import scrapy + +class UrlSpider(scrapy.Spider): + name = "{urlparse(url).netloc}" + + def parse(self, response): + return [{{"found_by_url": True}}] +""", + encoding="utf-8", + ) + returncode, out, err = proc("parse", url, cwd=proj_path) + assert returncode == 0, err + assert "Unable to find spider for" not in err + assert "{'found_by_url': True}" in out + + def test_legacy_item_processor( + self, proj_path: Path, mockserver: MockServer + ) -> None: + """--pipelines supports an ITEM_PROCESSOR without process_item_async().""" + (proj_path / self.project_name / "legacy.py").write_text( + """ +import logging + +from twisted.internet.defer import succeed + + +class LegacyItemProcessor: + @classmethod + def from_crawler(cls, crawler): + return cls() + + def open_spider(self, spider): + return succeed(None) + + def close_spider(self, spider): + return succeed(None) + + def process_item(self, item, spider): + logging.info("Legacy item processor!") + return succeed(item) +""", + encoding="utf-8", + ) + _, _, stderr = proc( + "parse", + "--spider", + self.spider_name, + "--pipelines", + "-c", + "parse", + "-s", + f"ITEM_PROCESSOR={self.project_name}.legacy.LegacyItemProcessor", + mockserver.url("/html"), + cwd=proj_path, + ) + assert "INFO: Legacy item processor!" in stderr + + @pytest.mark.parametrize("verbose", [True, False]) + def test_no_items_no_links( + self, verbose: bool, proj_path: Path, mockserver: MockServer + ) -> None: + args = ["--verbose"] if verbose else [] + _, out, err = proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse", + "--noitems", + "--nolinks", + *args, + mockserver.url("/html"), + cwd=proj_path, + ) + assert "# Scraped Items" not in out, err + assert "# Requests" not in out + def test_parse_add_options(self): command = parse.Command() command.settings = Settings() diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 2b410b5c6..11036eaeb 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -136,6 +136,12 @@ class MySpider(scrapy.Spider): log = self.get_log(tmp_path, "from scrapy.spiders import Spider\n") assert "No spider found in file" in log + @pytest.mark.parametrize("args", [(), ("a.py", "b.py")]) + def test_runspider_bad_arguments(self, args: tuple[str, ...]) -> None: + returncode, out, _ = proc("runspider", *args) + assert returncode == 2 + assert "Usage" in out + def test_runspider_file_not_found(self) -> None: _, _, log = proc("runspider", "some_non_existent_file") assert "File not found: some_non_existent_file" in log diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index f24200f53..29667a1ae 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -18,6 +18,7 @@ from scrapy.shell import Shell, inspect_response from scrapy.utils.reactor import _asyncio_reactor_path from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE, tests_datadir +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc from tests.utils.decorators import coroutine_test @@ -162,6 +163,33 @@ class TestShellCommand: assert ret == 0, out +class TestShellCommandWithSpider(TestProjectBase): + @pytest.fixture(autouse=True) + def create_files(self, proj_path: Path) -> None: + (proj_path / self.project_name / "spiders" / "myspider.py").write_text( + """ +import scrapy + +class MySpider(scrapy.Spider): + name = "myspider" +""", + encoding="utf-8", + ) + + def test_spider(self, proj_path: Path, mockserver: MockServer) -> None: + ret, out, err = proc( + "shell", + "--spider", + "myspider", + mockserver.url("/text"), + "-c", + "spider.name", + cwd=proj_path, + ) + assert ret == 0, err + assert out.strip() == "myspider" + + class TestInteractiveShell: def test_fetch(self, mockserver: MockServer) -> None: args = ( diff --git a/tests/test_commands.py b/tests/test_commands.py index 3e687e811..f20ecc153 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,21 +3,27 @@ from __future__ import annotations import argparse import json import sys +from pathlib import Path from typing import TYPE_CHECKING import pytest import scrapy from scrapy.cmdline import _pop_command_name, execute -from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.reactor import _asyncio_reactor_path from tests.utils.bases.commands import TestProjectBase -from tests.utils.cmdline import call, proc, write_recording_editor +from tests.utils.cmdline import ( + call, + proc, + write_recording_browser, + write_recording_editor, +) if TYPE_CHECKING: - from pathlib import Path + from tests.mockserver.http import MockServer class EmptyCommand(ScrapyCommand): @@ -107,6 +113,93 @@ class TestCommandSettings: ) +class TestGlobalOptions: + """Tests for the options that every command supports.""" + + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = "myspider" + + async def start(self): + self.logger.debug("It works!") + return + yield +""" + + @pytest.fixture + def spider_path(self, tmp_path: Path) -> Path: + path = tmp_path / "myspider.py" + path.write_text(self.spider_code, encoding="utf-8") + return path + + def test_invalid_set(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "-s", "FOO") + assert returncode == 2 + assert "Invalid -s value, use -s NAME=VALUE" in err + + def test_invalid_spider_argument(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "-a", "FOO") + assert returncode == 2 + assert "Invalid -a value, use -a NAME=VALUE" in err + + def test_logfile(self, tmp_path: Path, spider_path: Path) -> None: + logfile = tmp_path / "scrapy.log" + returncode, _, err = proc( + "runspider", str(spider_path), "--logfile", str(logfile) + ) + assert returncode == 0, err + assert "It works!" in logfile.read_text(encoding="utf-8") + assert "It works!" not in err + + def test_loglevel(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "--loglevel", "INFO") + assert returncode == 0, err + assert "It works!" not in err + assert "Spider closed (finished)" in err + + def test_nolog(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "--nolog") + assert returncode == 0, err + assert not err + + def test_pidfile(self, tmp_path: Path, spider_path: Path) -> None: + pidfile = tmp_path / "scrapy.pid" + returncode, _, err = proc( + "runspider", str(spider_path), "--pidfile", str(pidfile) + ) + assert returncode == 0, err + assert pidfile.read_text(encoding="utf-8").strip().isdigit() + + def test_pdb(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "--pdb") + assert returncode == 0, err + assert "It works!" in err + + +class TestSettingsCommand: + @pytest.mark.parametrize( + ("option", "setting", "expected"), + [ + ("--get", "BOT_NAME", "scrapybot"), + ("--getbool", "COOKIES_ENABLED", "True"), + ("--getint", "CONCURRENT_REQUESTS", "16"), + ("--getfloat", "DOWNLOAD_DELAY", "0.0"), + ("--getlist", "SPIDER_MODULES", "[]"), + ], + ) + def test_get(self, option: str, setting: str, expected: str) -> None: + returncode, out, err = proc("settings", option, setting) + assert returncode == 0, err + assert out.startswith(expected) + + def test_no_option(self) -> None: + returncode, out, err = proc("settings") + assert returncode == 0, err + assert not out + + class TestCommandCrawlerProcess(TestProjectBase): """Test that the command uses the expected kind of *CrawlerProcess and produces expected errors when needed.""" @@ -577,18 +670,31 @@ class TestBenchCommand: class TestViewCommand: - def test_methods(self) -> None: - command = view.Command() - command.settings = Settings() - parser = argparse.ArgumentParser( - prog="scrapy", - prefix_chars="-", - formatter_class=ScrapyHelpFormatter, - conflict_handler="resolve", + @pytest.mark.skipif( + sys.platform == "win32", reason="requires a POSIX shell browser script" + ) + def test_view( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mockserver: MockServer + ) -> None: + opened = tmp_path / "opened.txt" + browser = tmp_path / "fake-browser.sh" + write_recording_browser(browser, opened) + monkeypatch.setenv("BROWSER", str(browser)) + + returncode, _, err = proc("view", mockserver.url("/html"), cwd=tmp_path) + + assert returncode == 0, err + url = opened.read_text(encoding="utf-8") + assert url.startswith("file://") + body = Path(url.removeprefix("file://")).read_text(encoding="utf-8") + assert "

Works

" in body + + def test_non_text_response(self, mockserver: MockServer) -> None: + returncode, _, err = proc( + "view", mockserver.url("/static/files/images/scrapy.png") ) - command.add_options(parser) - assert command.short_desc() == "Open URL in browser, as seen by Scrapy" - assert "URL using the Scrapy downloader and show its" in command.long_desc() + assert returncode == 0, err + assert "Cannot view a non-text response." in err class TestEditCommand(TestProjectBase): @@ -615,6 +721,11 @@ class TestEditCommand(TestProjectBase): assert returncode == 1 assert "Spider not found: nonexistent" in err + def test_edit_no_spider(self, proj_path: Path) -> None: + returncode, out, _ = proc("edit", cwd=proj_path) + assert returncode == 2 + assert "Usage" in out + class TestHelpMessage(TestProjectBase): @pytest.mark.parametrize( diff --git a/tests/utils/cmdline.py b/tests/utils/cmdline.py index 62dff3d4c..095cb17a7 100644 --- a/tests/utils/cmdline.py +++ b/tests/utils/cmdline.py @@ -46,3 +46,16 @@ def write_recording_editor(editor: Path) -> None: open (its last argument) into the file given as its first argument.""" editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8") editor.chmod(0o755) + + +def write_recording_browser(browser: Path, recorded: Path) -> None: + """Create an executable browser script that writes the URL it is asked to + open into *recorded*. + + ``webbrowser`` only passes the URL to the command from the ``BROWSER`` + environment variable, hence the hardcoded output path. + """ + browser.write_text( + f'#!/bin/sh\nprintf "%s" "$1" > "{recorded}"\n', encoding="utf-8" + ) + browser.chmod(0o755) From aa5ded25398f9803b98d601439a4f0bf63f469f7 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 10:27:14 +0200 Subject: [PATCH 034/111] Set up CodSpeed (#7831) * Set up CodSpeed * CodSpeed: update permissions --- .github/workflows/codspeed.yml | 45 +++++++++++++++++ conftest.py | 3 ++ tests/benchmarks/__init__.py | 28 +++++++++++ tests/benchmarks/conftest.py | 27 ++++++++++ tests/benchmarks/test_benchmark_crawl.py | 64 ++++++++++++++++++++++++ tox.ini | 18 +++++++ 6 files changed, 185 insertions(+) create mode 100644 .github/workflows/codspeed.yml create mode 100644 tests/benchmarks/__init__.py create mode 100644 tests/benchmarks/conftest.py create mode 100644 tests/benchmarks/test_benchmark_crawl.py diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000..6930c4c1c --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,45 @@ +--- +name: codspeed + +on: + push: + branches: + - master + pull_request: + paths: + - scrapy/** + - tests/benchmarks/** + - .github/workflows/codspeed.yml + - tox.ini + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: {} + +jobs: + benchmark: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # OIDC authentication with CodSpeed + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python 3.14 + uses: actions/setup-python@v6 + with: + python-version: '3.14' + - name: Install dependencies + run: | + pip install --upgrade pip + pip install --upgrade tox + tox -n -e benchmark + - name: Run benchmarks + uses: CodSpeedHQ/action@v4 + with: + mode: simulation + run: tox -e benchmark diff --git a/conftest.py b/conftest.py index 27c398792..5a535c168 100644 --- a/conftest.py +++ b/conftest.py @@ -54,6 +54,9 @@ if not H2_ENABLED: if find_spec("httpx2") is None and find_spec("httpx") is None: collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py") +if find_spec("pytest_codspeed") is None: + collect_ignore.append("tests/benchmarks") + def pytest_addoption(parser, pluginmanager): if pluginmanager.hasplugin("twisted"): diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py new file mode 100644 index 000000000..7b5ca0cb9 --- /dev/null +++ b/tests/benchmarks/__init__.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from scrapy import Spider + from scrapy.crawler import Crawler + + +def crawl(spidercls: type[Spider], settings: dict[str, Any], **kwargs: Any) -> Crawler: + """Run a crawl to completion and return its crawler. + + Unlike the rest of the test suite, benchmarks run without ``pytest-twisted`` + and drive the reactor themselves, since the code being measured must be + callable synchronously by ``pytest-codspeed``. + """ + from twisted.internet import reactor + + crawler = get_crawler(spidercls, settings) + result: list[Any] = [] + crawler.crawl(**kwargs).addBoth(result.append) + while not result: + reactor.iterate(0.001) + if isinstance(result[0], BaseException): + raise result[0] + return crawler diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py new file mode 100644 index 000000000..55356083d --- /dev/null +++ b/tests/benchmarks/conftest.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from scrapy.utils.reactor import install_reactor + +if TYPE_CHECKING: + from collections.abc import Generator + + +@pytest.fixture(scope="session", autouse=True) +def running_reactor() -> Generator[None]: + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + + from twisted.internet import reactor + + # Marks the reactor as running without blocking, so that crawls can be + # driven with reactor.iterate(), see tests.benchmarks.crawl(). + reactor.startRunning(installSignalHandlers=False) + + yield + + reactor.stop() + # Lets the shutdown event triggers run, e.g. to join the thread pool. + reactor.iterate(0) diff --git a/tests/benchmarks/test_benchmark_crawl.py b/tests/benchmarks/test_benchmark_crawl.py new file mode 100644 index 000000000..4ad0e672b --- /dev/null +++ b/tests/benchmarks/test_benchmark_crawl.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any +from urllib.parse import urlencode + +import pytest + +from scrapy import Field, Item, Request, Spider +from scrapy.linkextractors import LinkExtractor +from tests.benchmarks import crawl + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found] + + from scrapy.http import Response + from tests.mockserver.http import MockServer + +pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed") + +PAGES = 100 +LINKS_PER_PAGE = 5 + + +class _Page(Item): + url = Field() + anchors = Field() + + +class _FollowSpider(Spider): + name = "benchmark" + url: str + link_extractor = LinkExtractor() + + async def start(self) -> AsyncIterator[Any]: + yield Request(self.url, dont_filter=True) + + def parse(self, response: Response) -> Any: + yield _Page( + url=response.url, + anchors=response.css("a::text").getall(), + ) + for link in self.link_extractor.extract_links(response): # type: ignore[arg-type] + yield Request(link.url) + + +class _Pipeline: + def process_item(self, item: Any) -> Any: + return item + + +def test_benchmark_crawl(benchmark: BenchmarkFixture, mockserver: MockServer) -> None: + """Crawl of a set of interlinked pages served over HTTP.""" + query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"}) + url = mockserver.url(f"/follow?{query}") + settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False} + + def run() -> None: + crawler = crawl(_FollowSpider, settings, url=url) + assert crawler.stats + assert crawler.stats.get_value("item_scraped_count") == PAGES + 1 + + benchmark(run) diff --git a/tox.ini b/tox.ini index e10a7cc0c..18e1579c9 100644 --- a/tox.ini +++ b/tox.ini @@ -30,6 +30,7 @@ envlist = botocore pypy3 pypy3-extra-deps + benchmark minversion = 1.7.0 [test-requirements] @@ -317,3 +318,20 @@ setenv = {[min]setenv} commands = pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore + + +# CPU benchmarks, tracked on CodSpeed. +# +# pytest-twisted is left out on purpose: benchmarked code must be callable +# synchronously, so tests/benchmarks drives the reactor itself. + +[testenv:benchmark] +basepython = python3.14 +deps = + pytest >= 8.4.1 + pytest-codspeed +passenv = + *codspeed* + *ci* +commands = + pytest {posargs:tests/benchmarks} --codspeed --codspeed-mode=simulation From 6cefaa5434da050bc23ffaaf2c1b38be1041bd31 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 12:43:40 +0200 Subject: [PATCH 035/111] Add a stats reference (#7814) --- .pre-commit-config.yaml | 2 +- docs/requirements.in | 2 +- docs/requirements.txt | 2 +- docs/topics/extensions.rst | 21 +- docs/topics/settings.rst | 3 +- docs/topics/stats.rst | 641 ++++++++++++++++++++++++++++++++++ scrapy/core/scheduler.py | 8 +- scrapy/extensions/logcount.py | 2 +- tox.ini | 2 +- 9 files changed, 659 insertions(+), 24 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 311df7052..c27348c7a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,6 @@ repos: hooks: - id: sphinx-lint - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.8.8 + rev: 0.8.9 hooks: - id: sphinx-scrapy diff --git a/docs/requirements.in b/docs/requirements.in index a1f3a7468..257365380 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -5,4 +5,4 @@ sphinx sphinx-notfound-page sphinx-rtd-theme sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9 diff --git a/docs/requirements.txt b/docs/requirements.txt index a5cbad302..87634cea9 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0 # via # -r docs/requirements.in # sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@912ed0507405e16ac60a47dd08195a1cd0ced984 # via -r docs/requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 5598ab983..78b38cc3f 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -136,18 +136,10 @@ Core Stats extension Enable the collection of core statistics, provided the stats collection is enabled (see :ref:`topics-stats`). -The following stats are collected: - -* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`). -* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`). -* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`). -* ``finish_reason``: the closing reason string (e.g. ``"finished"``, - ``"closespider_timeout"``). -* ``item_scraped_count``: total number of items that passed all pipelines. -* ``item_dropped_count``: total number of items dropped by a pipeline. -* ``item_dropped_reasons_count/``: per-exception drop count - (e.g. ``item_dropped_reasons_count/DropItem``). -* ``response_received_count``: total number of HTTP responses received. +The following stats are collected: :stat:`elapsed_time_seconds`, +:stat:`finish_reason`, :stat:`finish_time`, :stat:`item_dropped_count`, +:stat:`item_dropped_reasons_count/{exception}`, :stat:`item_scraped_count`, +:stat:`response_received_count`, :stat:`start_time`. Log Count extension ~~~~~~~~~~~~~~~~~~~ @@ -190,7 +182,7 @@ Monitors the memory used by the Scrapy process that runs the spider and: 1. sends a :signal:`memusage_warning_reached` signal when it exceeds :setting:`MEMUSAGE_WARNING_MB` -2. closes the spider with the `"memusage_exceeded"` reason when it exceeds +2. closes the spider with the ``"memusage_exceeded"`` reason when it exceeds :setting:`MEMUSAGE_LIMIT_MB` This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and @@ -214,7 +206,8 @@ An extension for debugging memory usage. It collects information about: * objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs` To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The -info will be stored in the stats. +info will be stored in the :stat:`memdebug/gc_garbage_count` and +:stat:`memdebug/live_refs/{cls}` stats. .. _topics-extensions-ref-spiderstate: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 81055afc2..b07dff180 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1871,7 +1871,8 @@ Default: ``False`` Setting to ``True`` will log debug information about the requests scheduler. This currently logs (only once) if the requests cannot be serialized to disk. -Stats counter (``scheduler/unserializable``) tracks the number of times this happens. +The :stat:`scheduler/unserializable` stat tracks the number of times this +happens. Example entry in logs:: diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 0cf4a72cc..c702cefe7 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -21,6 +21,8 @@ using the Stats Collector from. Another feature of the Stats Collector is that it's very efficient (when enabled) and extremely efficient (almost unnoticeable) when disabled. +See :ref:`topics-stats-reference` below for the stats that Scrapy sets. + .. _topics-stats-usecases: Common Stats Collector uses @@ -101,3 +103,642 @@ DummyStatsCollector ------------------- .. autoclass:: DummyStatsCollector + +.. _topics-stats-reference: + +Built-in stats reference +======================== + +Scrapy sets the following :ref:`stats `. Components other than +those built into Scrapy may set additional stats; see their documentation. + +Stat keys that contain a ``{placeholder}`` below stand for a family of stats, +one per actual value of the placeholder. + +.. note:: Most stats are set by a specific :ref:`component + `, and are only present if that component is enabled and + its code path is reached. A stat that is missing from + :meth:`~scrapy.statscollectors.StatsCollector.get_stats` output is + equivalent to a counter of 0. + +.. stat:: downloader/exception_count + +``downloader/exception_count`` + Number of exceptions raised while downloading requests. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/exception_type_count/{exception_type} + +``downloader/exception_type_count/{exception_type}`` + Number of exceptions raised while downloading requests, per exception type, + where ``{exception_type}`` is the import path of the exception class, e.g. + ``twisted.internet.error.DNSLookupError``. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_bytes + +``downloader/request_bytes`` + Total size, in bytes, of the requests sent, counting the request line, the + headers and the body. As with :stat:`downloader/request_count`, requests + served from the cache are also counted. + + It is an approximation, reconstructed from each :class:`~scrapy.Request` + object instead of measured on the wire, so it does not account for the + actual bytes that the :ref:`download handler + ` sends, e.g. transport-level overhead. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_count + +``downloader/request_count`` + Number of requests sent. + + Requests that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache are also counted, even though they are never sent, + because it handles requests after + :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_method_count/{method} + +``downloader/request_method_count/{method}`` + Number of requests sent, per HTTP method, e.g. ``GET`` or ``POST``. As with + :stat:`downloader/request_count`, requests served from the cache are also + counted. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_bytes + +``downloader/response_bytes`` + Total size, in bytes, of the responses received, counting the status line, + the headers and the body. It covers the same responses as + :stat:`downloader/response_count`. + + The body is counted as received, i.e. still compressed for responses that + used ``Content-Encoding``, because + :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` handles + responses before + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + decompresses them. See :stat:`httpcompression/response_bytes` for + decompressed sizes. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_count + +``downloader/response_count`` + Number of responses received. + + It counts responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache, even though they do not come from the network, and + responses that a downloader middleware consumes before they reach your + spider, e.g. redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` + turns into new requests. Compare with :stat:`response_received_count`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_status_count/{status_code} + +``downloader/response_status_count/{status_code}`` + Number of responses received, per HTTP status code, e.g. ``200`` or + ``404``. It covers the same responses as :stat:`downloader/response_count`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: dupefilter/filtered + +``dupefilter/filtered`` + Number of requests dropped as duplicates. + + Set by :class:`~scrapy.dupefilters.RFPDupeFilter`. + +.. stat:: elapsed_time_seconds + +``elapsed_time_seconds`` + Time, as a :class:`float`, in seconds, between the :signal:`spider_opened` + and the :signal:`spider_closed` signals. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: feedexport/failed_count/{storage} + +``feedexport/failed_count/{storage}`` + Number of :ref:`feeds ` that could not be stored, per + :ref:`storage backend `, where ``{storage}`` + is the class name of the storage backend, e.g. ``FileFeedStorage``. + +.. stat:: feedexport/success_count/{storage} + +``feedexport/success_count/{storage}`` + Number of :ref:`feeds ` stored successfully, per + :ref:`storage backend `, where ``{storage}`` + is the class name of the storage backend, e.g. ``FileFeedStorage``. + +.. stat:: file_count + +``file_count`` + Number of files handled by the :ref:`media pipelines + `. + +.. stat:: file_status_count/{status} + +``file_status_count/{status}`` + Number of files handled by the :ref:`media pipelines + `, per status, where ``{status}`` is one of: + + - ``downloaded``: the file was downloaded. + + - ``cached``: the file came from the + :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + cache. + + - ``uptodate``: the file was already in the storage backend and had not + :ref:`expired `, so it was not downloaded again. + +.. stat:: finish_reason + +``finish_reason`` + String indicating why the crawl finished. It matches the *reason* argument + of the :signal:`spider_closed` signal. + + Scrapy uses the following reasons: + + - ``cancelled``: the spider was closed without a more specific reason, + e.g. because :exc:`~scrapy.exceptions.CloseSpider` was raised without + one. + + - ``closespider_errorcount``: see :setting:`CLOSESPIDER_ERRORCOUNT`. + + - ``closespider_itemcount``: see :setting:`CLOSESPIDER_ITEMCOUNT`. + + - ``closespider_pagecount``: see :setting:`CLOSESPIDER_PAGECOUNT`. + + - ``closespider_pagecount_no_item``: see + :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM`. + + - ``closespider_timeout``: see :setting:`CLOSESPIDER_TIMEOUT`. + + - ``closespider_timeout_no_item``: see + :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`. + + - ``finished``: the spider became idle with no pending requests, i.e. it + finished normally. + + - ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`. + + - ``shutdown``: the crawl was interrupted, e.g. by a system signal such + as ``SIGINT`` (:kbd:`Ctrl-C`). + + Third-party components and your own code may use any other reason, e.g. by + raising :exc:`~scrapy.exceptions.CloseSpider` with it. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: finish_time + +``finish_time`` + Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when + the :signal:`spider_closed` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: httpcache/errorrecovery + +``httpcache/errorrecovery`` + Number of times that a stale cached response was used because downloading a + fresh response raised an exception. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/firsthand + +``httpcache/firsthand`` + Number of responses that were downloaded without a matching cache entry to + validate against, i.e. responses for requests counted in + :stat:`httpcache/miss`. + + It is lower than :stat:`httpcache/miss` when some of those requests yield + no response, either because they are dropped (see + :stat:`httpcache/ignore`) or because their download fails. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/hit + +``httpcache/hit`` + Number of requests served from the cache. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/ignore + +``httpcache/ignore`` + Number of requests dropped because they were not in the cache and + :setting:`HTTPCACHE_IGNORE_MISSING` is ``True``. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/invalidate + +``httpcache/invalidate`` + Number of times that a cached response failed validation and was replaced + with a freshly downloaded response. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/miss + +``httpcache/miss`` + Number of requests for which no cache entry could be read, either because + there was none or because reading it failed, in which case the request is + also counted in :stat:`httpcache/retrieve_error`. Those requests are + downloaded (see :stat:`httpcache/firsthand`), or dropped if + :setting:`HTTPCACHE_IGNORE_MISSING` is ``True`` (see + :stat:`httpcache/ignore`). + + Requests with a stale cache entry are not counted here; see + :stat:`httpcache/revalidate` and :stat:`httpcache/invalidate`. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/retrieve_error + +``httpcache/retrieve_error`` + Number of cache entries that could not be read, and hence were treated as + cache misses. Those requests are also counted in :stat:`httpcache/miss`. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/revalidate + +``httpcache/revalidate`` + Number of times that a cached response was successfully validated against + the target server, and hence used instead of the fresh response. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/store + +``httpcache/store`` + Number of responses stored in the cache. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/uncacheable + +``httpcache/uncacheable`` + Number of responses not stored in the cache because the + :setting:`HTTPCACHE_POLICY` did not allow it. + + Every response considered for caching is counted either here or in + :stat:`httpcache/store`, so ``httpcache/store + httpcache/uncacheable`` + equals ``httpcache/firsthand + httpcache/invalidate``. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcompression/response_bytes + +``httpcompression/response_bytes`` + Total size, in bytes, of decompressed response bodies, counting only the + body and only responses that were actually decompressed. Compare with + :stat:`downloader/response_bytes`. + + Set by + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`. + +.. stat:: httpcompression/response_count + +``httpcompression/response_count`` + Number of decompressed responses. + + Set by + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`. + +.. stat:: httperror/response_ignored_count + +``httperror/response_ignored_count`` + Number of responses dropped because of their HTTP status code. + + Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`. + +.. stat:: httperror/response_ignored_status_count/{status_code} + +``httperror/response_ignored_status_count/{status_code}`` + Number of responses dropped because of their HTTP status code, per HTTP + status code, e.g. ``404``. + + Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`. + +.. stat:: item_dropped_count + +``item_dropped_count`` + Number of items dropped by an :ref:`item pipeline + `, i.e. number of times that the + :signal:`item_dropped` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: item_dropped_reasons_count/{exception} + +``item_dropped_reasons_count/{exception}`` + Number of items dropped, per exception, where ``{exception}`` is the class + name of the exception that caused the item to be dropped. + + Only :exc:`~scrapy.exceptions.DropItem` and its subclasses drop items, and + each one is counted under its own class name, e.g. + ``item_dropped_reasons_count/DropItem`` for + :exc:`~scrapy.exceptions.DropItem` itself and + ``item_dropped_reasons_count/MyDropItem`` for a ``MyDropItem`` subclass of + it. Any other exception raised by an :ref:`item pipeline + ` triggers the :signal:`item_error` signal instead of + :signal:`item_dropped`, and is not counted here or in + :stat:`item_dropped_count`. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: item_scraped_count + +``item_scraped_count`` + Number of items that passed all :ref:`item pipelines + `, i.e. number of times that the + :signal:`item_scraped` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: items_per_minute + +``items_per_minute`` + Average number of items scraped per minute during the crawl. + + It is ``None`` if the crawl took less than a minute. + + Set by :class:`~scrapy.extensions.logstats.LogStats`. + +.. stat:: log_count/{level} + +``log_count/{level}`` + Number of log messages, per logging level name, e.g. ``INFO`` or + ``WARNING``. + + Only messages that the :setting:`LOG_LEVEL` setting allows are counted. + + Set by :class:`~scrapy.extensions.logcount.LogCount`. + +.. stat:: memdebug/gc_garbage_count + +``memdebug/gc_garbage_count`` + Number of objects in :data:`gc.garbage` when the spider is closed. + + Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires + :setting:`MEMDEBUG_ENABLED` to be ``True``. + +.. stat:: memdebug/live_refs/{cls} + +``memdebug/live_refs/{cls}`` + Number of live objects of class ``{cls}`` when the spider is closed, as + reported by :ref:`trackref `, e.g. + ``memdebug/live_refs/HtmlResponse``. + + Only set for classes with at least 1 live object. + + Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires + :setting:`MEMDEBUG_ENABLED` to be ``True``. + +.. stat:: memusage/limit_reached + +``memusage/limit_reached`` + ``1`` if memory usage exceeded :setting:`MEMUSAGE_LIMIT_MB`, which also + stops the crawl. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/max + +``memusage/max`` + Maximum peak memory usage, in bytes, observed during the crawl. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/startup + +``memusage/startup`` + Peak memory usage, in bytes, when the engine started. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/warning_reached + +``memusage/warning_reached`` + ``1`` if memory usage exceeded :setting:`MEMUSAGE_WARNING_MB`. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: offsite/domains + +``offsite/domains`` + Number of distinct domains for which at least 1 request was dropped for + being offsite. + + Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`. + +.. stat:: offsite/filtered + +``offsite/filtered`` + Number of requests dropped for being offsite. + + Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`. + +.. stat:: request_depth_count/{depth} + +``request_depth_count/{depth}`` + Number of requests scheduled at depth ``{depth}``, e.g. + ``request_depth_count/2``. + + Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`, which + requires :setting:`DEPTH_STATS_VERBOSE` to be ``True`` for this stat. + +.. stat:: request_depth_max + +``request_depth_max`` + Maximum depth reached. + + Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`. + +.. stat:: response_received_count + +``response_received_count`` + Number of responses received, i.e. number of times that the + :signal:`response_received` signal was sent. + + Unlike :stat:`downloader/response_count`, it does not count responses that + a downloader middleware consumes before they reach the engine, e.g. + redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` + turns into new requests. Both count responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: responses_per_minute + +``responses_per_minute`` + Average number of responses received per minute during the crawl. + + It is ``None`` if the crawl took less than a minute. + + Set by :class:`~scrapy.extensions.logstats.LogStats`. + +.. stat:: retry/count + +``retry/count`` + Number of requests retried. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. stat:: retry/max_reached + +``retry/max_reached`` + Number of requests that were not retried because they had already been + retried :setting:`RETRY_TIMES` times. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. stat:: retry/reason_count/{reason} + +``retry/reason_count/{reason}`` + Number of requests retried, per reason, e.g. + ``retry/reason_count/twisted.internet.error.TimeoutError`` or + ``retry/reason_count/504 Gateway Time-out``. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. note:: Code calling + :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` may pass a + custom *stats_base_key*, in which case ``retry`` is replaced with that key + in the 3 stats above. + +.. stat:: robotstxt/exception_count/{exception_type} + +``robotstxt/exception_count/{exception_type}`` + Number of exceptions raised while downloading ``robots.txt`` files, per + exception type, where ``{exception_type}`` is the string representation of + the exception class, e.g. ````. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/forbidden + +``robotstxt/forbidden`` + Number of requests dropped for being disallowed by ``robots.txt``. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/request_count + +``robotstxt/request_count`` + Number of ``robots.txt`` files requested, i.e. 1 per network location for + which at least 1 request was sent. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/response_count + +``robotstxt/response_count`` + Number of ``robots.txt`` responses received. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/response_status_count/{status_code} + +``robotstxt/response_status_count/{status_code}`` + Number of ``robots.txt`` responses received, per HTTP status code, e.g. + ``404``. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: scheduler/dequeued + +``scheduler/dequeued`` + Number of requests read from the :ref:`scheduler `. + +.. stat:: scheduler/dequeued/disk + +``scheduler/dequeued/disk`` + Number of requests read from the disk queue of the :ref:`scheduler + `. + +.. stat:: scheduler/dequeued/memory + +``scheduler/dequeued/memory`` + Number of requests read from the memory queue of the :ref:`scheduler + `. + +.. stat:: scheduler/enqueued + +``scheduler/enqueued`` + Number of requests stored into the :ref:`scheduler `. + +.. stat:: scheduler/enqueued/disk + +``scheduler/enqueued/disk`` + Number of requests stored into the disk queue of the :ref:`scheduler + `. + +.. stat:: scheduler/enqueued/memory + +``scheduler/enqueued/memory`` + Number of requests stored into the memory queue of the :ref:`scheduler + `. + +.. stat:: scheduler/unserializable + +``scheduler/unserializable`` + Number of requests that could not be stored into the disk queue of the + :ref:`scheduler ` because they could not be + :ref:`serialized `, and hence were stored into the + memory queue instead. + +.. stat:: spider_exceptions/count + +``spider_exceptions/count`` + Number of unhandled exceptions raised by spider callbacks. + + Set by the :ref:`scraper `. + +.. stat:: spider_exceptions/{exception} + +``spider_exceptions/{exception}`` + Number of unhandled exceptions raised by spider callbacks, per exception, + where ``{exception}`` is the class name of the exception, e.g. + ``spider_exceptions/ValueError``. + + Set by the :ref:`scraper `. + +.. stat:: start_time + +``start_time`` + Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when + the :signal:`spider_opened` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: urllength/request_ignored_count + +``urllength/request_ignored_count`` + Number of requests dropped for having a URL longer than + :setting:`URLLENGTH_LIMIT`. + + Set by :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`. diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 42c517222..a511b0c7b 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -366,8 +366,8 @@ class Scheduler(BaseScheduler): Unless the received request is filtered out by the Dupefilter, attempt to push it into the disk queue, falling back to pushing it into the memory queue. - Increment the appropriate stats, such as: ``scheduler/enqueued``, - ``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``. + Increment the appropriate stats, such as: :stat:`scheduler/enqueued`, + :stat:`scheduler/enqueued/disk`, :stat:`scheduler/enqueued/memory`. Return ``True`` if the request was stored successfully, ``False`` otherwise. """ @@ -390,8 +390,8 @@ class Scheduler(BaseScheduler): falling back to the disk queue if the memory queue is empty. Return ``None`` if there are no more enqueued requests. - Increment the appropriate stats, such as: ``scheduler/dequeued``, - ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. + Increment the appropriate stats, such as: :stat:`scheduler/dequeued`, + :stat:`scheduler/dequeued/disk`, :stat:`scheduler/dequeued/memory`. """ request: Request | None = self.mqs.pop() assert self.stats is not None diff --git a/scrapy/extensions/logcount.py b/scrapy/extensions/logcount.py index e6d51a7d8..fcce64438 100644 --- a/scrapy/extensions/logcount.py +++ b/scrapy/extensions/logcount.py @@ -20,7 +20,7 @@ class LogCount: """Install a log handler that counts log messages by level. The handler installed is :class:`scrapy.utils.log.LogCounterHandler`. - The counts are stored in stats as ``log_count/``. + The counts are stored in the :stat:`log_count/{level}` stat. .. versionadded:: 2.14 """ diff --git a/tox.ini b/tox.ini index 18e1579c9..b59221340 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] requires = - sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 + sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9 envlist = pre-commit pylint From 6f87d3f86334855484c8e1b0888976c70596b0e9 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 13:46:33 +0200 Subject: [PATCH 036/111] Rename the current benchmark for the future (#7839) --- .../{test_benchmark_crawl.py => test_crawl.py} | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) rename tests/benchmarks/{test_benchmark_crawl.py => test_crawl.py} (82%) diff --git a/tests/benchmarks/test_benchmark_crawl.py b/tests/benchmarks/test_crawl.py similarity index 82% rename from tests/benchmarks/test_benchmark_crawl.py rename to tests/benchmarks/test_crawl.py index 4ad0e672b..0fdfe742b 100644 --- a/tests/benchmarks/test_benchmark_crawl.py +++ b/tests/benchmarks/test_crawl.py @@ -50,8 +50,13 @@ class _Pipeline: return item -def test_benchmark_crawl(benchmark: BenchmarkFixture, mockserver: MockServer) -> None: - """Crawl of a set of interlinked pages served over HTTP.""" +def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None: + """Per-request overhead of a crawl over HTTP. + + The pages are small on purpose, so that the cost of parsing them stays + negligible next to the cost of moving requests and responses through the + engine, the middlewares and the download handler. + """ query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"}) url = mockserver.url(f"/follow?{query}") settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False} From 1f03fbc17e1bf7ecdd7b7df06d7e3cf8c4f654f7 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 15:59:38 +0200 Subject: [PATCH 037/111] Add scrapy.utils.asyncio.sleep() (#7843) --- docs/topics/asyncio.rst | 1 + scrapy/utils/asyncio.py | 20 +++++++++++++++++++- scrapy/utils/defer.py | 11 ++--------- tests/test_crawler_subprocess.py | 5 +++-- tests/test_engine_loop.py | 11 +++++------ tests/test_utils_asyncio.py | 11 +++++++++++ tests/utils/__init__.py | 10 ---------- 7 files changed, 41 insertions(+), 28 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 63c217e93..afccb491d 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -267,6 +267,7 @@ Here are some examples of APIs and patterns that need a replacement: Scrapy provides unified helpers for some of these examples: +.. autofunction:: scrapy.utils.asyncio.sleep .. autofunction:: scrapy.utils.asyncio.call_later .. autofunction:: scrapy.utils.asyncio.create_looping_call .. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 44604c0fe..7c7697f56 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar from twisted.internet.defer import Deferred -from twisted.internet.task import LoopingCall +from twisted.internet.task import LoopingCall, deferLater from twisted.internet.threads import deferToThread from scrapy.utils.asyncgen import as_async_generator @@ -293,6 +293,24 @@ class CallLaterResult: self._delayed_call = None +async def sleep(seconds: float) -> None: + """Sleep for *seconds*. + + .. versionadded:: VERSION + + This uses either :func:`asyncio.sleep` or + :func:`~twisted.internet.task.deferLater`, depending on whether asyncio + support is available. + """ + if is_asyncio_available(): + await asyncio.sleep(seconds) + return + + from twisted.internet import reactor + + await deferLater(reactor, seconds) + + async def run_in_thread( func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs ) -> _T: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d0259b634..7c6235f29 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -27,7 +27,7 @@ from twisted.internet.task import Cooperator from twisted.python import failure from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.asyncio import is_asyncio_available, sleep from scrapy.utils.python import global_object_name if TYPE_CHECKING: @@ -90,14 +90,7 @@ async def _defer_sleep_async() -> None: """Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ - if is_asyncio_available(): - await asyncio.sleep(_DEFER_DELAY) - else: - from twisted.internet import reactor - - d: Deferred[None] = Deferred() - reactor.callLater(_DEFER_DELAY, d.callback, None) - await d + await sleep(_DEFER_DELAY) def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index fe2f83161..733b6797d 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -14,7 +14,8 @@ from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn from w3lib import __version__ as w3lib_version -from tests.utils import async_sleep, get_script_run_env +from scrapy.utils.asyncio import sleep +from tests.utils import get_script_run_env from tests.utils.decorators import coroutine_test if TYPE_CHECKING: @@ -244,7 +245,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): p.kill(sig) p.expect_exact("shutting down gracefully") # sending the second signal too fast often causes problems - await async_sleep(0.01) + await sleep(0.01) p.kill(sig) p.expect_exact("forcing unclean shutdown") p.wait() # type: ignore[no-untyped-call] diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index c15c396d3..14ec3d184 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -6,10 +6,9 @@ from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.core.scheduler import BaseScheduler -from scrapy.utils.asyncio import call_later +from scrapy.utils.asyncio import call_later, sleep from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer -from tests.utils import async_sleep from tests.utils.decorators import coroutine_test if TYPE_CHECKING: @@ -65,23 +64,23 @@ class TestMain: async def start(self): yield Request("data:,a") - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.pause() self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) # During this time, the scheduler reports having requests but # returns None. - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.unpause() # The scheduler request is processed. - await async_sleep(seconds) + await sleep(seconds) yield Request("data:,c") - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.pause() self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d")) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 9b7eb22fa..4bd54acd4 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -12,7 +12,9 @@ from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.asyncio import ( AsyncioLoopingCall, _parallel_asyncio, + call_later, is_asyncio_available, + sleep, ) from tests.utils.decorators import coroutine_test @@ -26,6 +28,15 @@ async def test_is_asyncio_available(reactor_pytest: str) -> None: assert is_asyncio_available() == (reactor_pytest != "default") +@coroutine_test +async def test_sleep() -> None: + events: list[str] = [] + call_later(0.05, events.append, "call_later") + await sleep(0.1) + events.append("sleep") + assert events == ["call_later", "sleep"] + + @pytest.mark.only_asyncio class TestParallelAsyncio: """Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync.""" diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index b0632a7ea..b27c5ade7 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import os from pathlib import Path from typing import TYPE_CHECKING @@ -8,8 +7,6 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred from scrapy.settings import Settings, default_settings -from scrapy.utils.asyncio import is_asyncio_available -from scrapy.utils.defer import maybe_deferred_to_future if TYPE_CHECKING: from collections.abc import Callable @@ -23,13 +20,6 @@ def twisted_sleep(seconds: float): return d -async def async_sleep(seconds: float) -> None: - if is_asyncio_available(): - await asyncio.sleep(seconds) - else: - await maybe_deferred_to_future(twisted_sleep(seconds)) - - def get_script_run_env() -> dict[str, str]: """Return a OS environment dict suitable to run scripts shipped with tests.""" From 14478e3f24258ad3e9b5a3ca8a178ef126c2d4c4 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 17:19:07 +0200 Subject: [PATCH 038/111] Support CONCURRENT_REQUESTS = 0 for unlimited concurrency (#7840) --- docs/topics/settings.rst | 2 +- scrapy/core/downloader/__init__.py | 3 ++- scrapy/core/downloader/handlers/_httpx.py | 7 ++++--- tests/test_core_downloader.py | 18 ++++++++++++++++++ tests/test_downloader_handler_httpx.py | 14 ++++++++++++++ 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index b07dff180..e287c3bd5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -575,7 +575,7 @@ CONCURRENT_REQUESTS Default: ``16`` The maximum number of concurrent (i.e. simultaneous) requests that will be -performed by the Scrapy downloader. +performed by the Scrapy downloader. Use ``0`` for no limit. .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7c0ee0eec..eb2079d0c 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -138,7 +138,8 @@ class Downloader: self.active.remove(request) def needs_backout(self) -> bool: - return len(self.active) >= self.total_concurrency + # A total concurrency of 0 means no limit. + return 0 < self.total_concurrency <= len(self.active) @_warn_spider_arg def _get_slot( diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index d5a4e9fcd..8bbffb233 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -92,10 +92,11 @@ class HttpxDownloadHandler(_Base): self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings) self._bind_host: str | None = self._get_bind_address_host() self._limits: httpx.Limits = httpx.Limits( - # hard limit on simultaneous connections - max_connections=self._pool_size_total, + # hard limit on simultaneous connections (None for no limit, which + # is what a CONCURRENT_REQUESTS of 0 means) + max_connections=self._pool_size_total or None, # total number of idle connections in the pool (extra ones are closed) - max_keepalive_connections=self._pool_size_total, + max_keepalive_connections=self._pool_size_total or None, ) self._default_client: httpx.AsyncClient = self._make_client() diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index fdd5edc27..3e4139b3e 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -14,6 +14,7 @@ from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse +from scrapy import Request from scrapy.core.downloader import Downloader, Slot, tls from scrapy.core.downloader.contextfactory import ( _load_context_factory_from_settings, @@ -296,6 +297,23 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): await self._assert_factory_works(server_url, client_context_factory) +@pytest.mark.parametrize( + ("concurrency", "active", "expected"), + [ + (2, 1, False), + (2, 2, True), + (0, 0, False), + (0, 2, False), + ], +) +def test_needs_backout(concurrency: int, active: int, expected: bool) -> None: + crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency}) + downloader = Downloader(crawler) + downloader.active = {Request(f"https://example.com/{i}") for i in range(active)} + assert downloader.needs_backout() is expected + downloader.close() + + @coroutine_test async def test_fetch_deprecated_spider_arg(): class CustomDownloader(Downloader): diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index 5ceb93382..976daacaf 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -15,6 +15,8 @@ from scrapy.core.downloader.handlers._httpx import ( HttpxDownloadHandler, ) from scrapy.exceptions import DownloadFailedError +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler from tests.utils.bases.download_handlers_http import ( TestHttpBase, TestHttpProxyBase, @@ -161,3 +163,15 @@ class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase): @pytest.mark.requires_internet class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase): pass + + +@pytest.mark.parametrize(("concurrency", "expected"), [(16, 16), (0, None)]) +@coroutine_test +async def test_pool_limits(concurrency: int, expected: int | None) -> None: + crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency}) + handler = build_from_crawler(HttpxDownloadHandler, crawler) + try: + assert handler._limits.max_connections == expected + assert handler._limits.max_keepalive_connections == expected + finally: + await handler.close() From 8caaac6ecbd49ad950b7a665498ccebe9fbff052 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:05:22 -0300 Subject: [PATCH 039/111] fix(shell): Run IPython prompt in a thread under a running loop (#7816) --- pyproject.toml | 4 +- scrapy/utils/console.py | 28 ++++++----- tests/test_utils_console.py | 98 +++++++++++++++++++++++++++++++++++++ tox.ini | 4 +- 4 files changed, 119 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 576a42e5c..1cbd39946 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,8 +69,8 @@ brotli = [ gcs = ["google-cloud-storage>=1.29.0"] httpx = ["httpx2[http2,socks]>=2.0.0"] images = ["Pillow>=8.3.2"] -ipython = ["ipython>=7.1.0"] -ptpython = ["ptpython>=2.0.1"] +ipython = ["ipython>=8.15.0"] +ptpython = ["ptpython>=3.0.23"] robotparser = ["robotexclusionrulesparser>=1.6.2"] s3 = ["boto3>=1.20.0"] twisted-http2 = ["Twisted[http2]>=21.7.0"] diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 31a4bb32f..23b4401e8 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,8 +1,9 @@ from __future__ import annotations +import asyncio import code from collections.abc import Callable -from functools import wraps +from functools import partial, wraps from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -16,16 +17,8 @@ def _embed_ipython_shell( namespace: dict[str, Any] | None = None, banner: str = "" ) -> EmbedFuncT: """Start an IPython Shell""" - try: - from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415 - from IPython.terminal.ipapp import load_default_config # noqa: PLC0415 - except ImportError: - from IPython.frontend.terminal.embed import ( # type: ignore[import-not-found,no-redef] # noqa: T100,PLC0415 - InteractiveShellEmbed, - ) - from IPython.frontend.terminal.ipapp import ( # type: ignore[import-not-found,no-redef] # noqa: PLC0415 - load_default_config, - ) + from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415 + from IPython.terminal.ipapp import load_default_config # noqa: PLC0415 @wraps(_embed_ipython_shell) def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: @@ -38,6 +31,19 @@ def _embed_ipython_shell( shell = InteractiveShellEmbed.instance( banner1=banner, user_ns=namespace, config=config ) + # If an asyncio event loop is already running in this thread, e.g. when + # inspect_response() is called from a spider callback while using the + # asyncio reactor, prompt_toolkit cannot run its own event loop here, so + # ask it to run the prompt in a separate thread instead. pt_app is None + # when IPython falls back to its simple prompt, which needs no event loop. + # See https://github.com/scrapy/scrapy/issues/5447 + if (pt_app := getattr(shell, "pt_app", None)) is not None: + try: + asyncio.get_running_loop() + except RuntimeError: + pass + else: + pt_app.prompt = partial(pt_app.prompt, in_thread=True) shell() return wrapper diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index ab0c72d8a..0dea8af6d 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -1,10 +1,38 @@ from __future__ import annotations +import subprocess +import sys from importlib.util import find_spec +from io import BytesIO +from typing import TYPE_CHECKING import pytest +from pexpect import EOF from scrapy.utils.console import get_shell_embed_func, start_python_console +from scrapy.utils.test import get_testenv + +if TYPE_CHECKING: + from pathlib import Path + +CONSOLE = """ +from scrapy.utils.console import start_python_console + +start_python_console(banner="SHELL-READY", shells=["ipython"]) +""" + +CONSOLE_IN_RUNNING_LOOP = """ +import asyncio + +from scrapy.utils.console import start_python_console + + +async def main(): + start_python_console(banner="SHELL-READY", shells=["ipython"]) + + +asyncio.run(main()) +""" def test_get_shell_embed_func(): @@ -61,6 +89,76 @@ def test_get_shell_embed_func_default(): assert shell.__name__ == expected +@pytest.mark.skipif(find_spec("IPython") is None, reason="IPython is not installed") +class TestIPythonShell: + """Starting an IPython shell, with and without an asyncio event loop already + running in the calling thread. The latter happens when inspect_response() is + called from a spider callback while using the asyncio reactor.""" + + @staticmethod + def _env(tmp_path: Path) -> dict[str, str]: + env = get_testenv() + # Keep IPython away from the profile and history of the user running the tests. + env["IPYTHONDIR"] = str(tmp_path) + return env + + def test_simple_prompt(self, tmp_path: Path) -> None: + """IPython falls back to its simple prompt, which needs no event loop, + when stdin is not a TTY.""" + env = self._env(tmp_path) + p = subprocess.run( + [sys.executable, "-c", CONSOLE_IN_RUNNING_LOOP], + check=False, + capture_output=True, + encoding="utf-8", + timeout=60, + env=env, + stdin=subprocess.DEVNULL, + ) + output = p.stdout + p.stderr + assert "SHELL-READY" in output + assert p.returncode == 0, output + + @pytest.mark.skipif( + sys.platform == "win32", reason="requires a POSIX pseudo-terminal" + ) + @pytest.mark.parametrize( + "script", + [CONSOLE, CONSOLE_IN_RUNNING_LOOP], + ids=["no_running_loop", "running_loop"], + ) + def test_tty(self, tmp_path: Path, script: str) -> None: + """IPython uses prompt_toolkit, which needs an event loop of its own, + when stdin is a TTY.""" + # pexpect only defines spawn, which needs a pseudo-terminal, on POSIX. + from pexpect import spawn # noqa: PLC0415 + + env = self._env(tmp_path) + env.pop("IPY_TEST_SIMPLE_PROMPT", None) + env["TERM"] = "xterm" + logfile = BytesIO() + p = spawn( + sys.executable, + ["-c", script], + env=env, + timeout=60, + ) + p.logfile_read = logfile + try: + # Wait for the prompt, which prompt_toolkit draws once it is done + # querying the terminal, before typing into it. + p.expect(r"In \[") + p.sendline("21*2") + p.expect_exact("42") + p.sendline("exit()") + p.expect(EOF) + finally: + p.close() + output = logfile.getvalue().decode() + assert "Traceback" not in output + assert p.exitstatus == 0, output + + def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None: def embed(namespace: dict[str, object], banner: str) -> None: raise SystemExit diff --git a/tox.ini b/tox.ini index b59221340..8331e2ab6 100644 --- a/tox.ini +++ b/tox.ini @@ -190,8 +190,8 @@ deps = brotlicffi==1.2.0.0; implementation_name == "pypy" google-cloud-storage==1.29.0 httpx2[http2,socks]==2.0.0 - ipython==7.1.0 - ptpython==2.0.1 + ipython==8.15.0 + ptpython==3.0.23 robotexclusionrulesparser==1.6.2 uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy" zstandard==0.16.0; implementation_name != "pypy" From 24de06bcd3bcf20b787e33b1f65a3512a226d1d9 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 20:02:52 +0200 Subject: [PATCH 040/111] CI: Install dependencies with uv (#7838) * CI: Install dependencies with uv * Setup GitHub Actions hardening * CI: Use uv for mitmproxy, benchmarks and cache keys * Fix mitmproxy install on PyPy --- .github/dependabot.yml | 12 +++++++++ .github/workflows/auto-close-llm-pr.yml | 6 +++-- .github/workflows/checks.yml | 32 +++++++++++++++++----- .github/workflows/codspeed.yml | 24 +++++++++++++---- .github/workflows/publish.yml | 35 +++++++++++++++++++----- .github/workflows/tests-macos.yml | 35 +++++++++++++++++++----- .github/workflows/tests-ubuntu.yml | 36 +++++++++++++++++++------ .github/workflows/tests-windows.yml | 35 +++++++++++++++++++----- .pre-commit-config.yaml | 7 ++++- docs/requirements.in | 2 +- docs/requirements.txt | 2 +- tox.ini | 3 ++- 12 files changed, 182 insertions(+), 47 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..623d48cfa --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: monthly + groups: + github-actions: + patterns: + - "*" + cooldown: + default-days: 7 diff --git a/.github/workflows/auto-close-llm-pr.yml b/.github/workflows/auto-close-llm-pr.yml index 160b39488..15120b0d9 100644 --- a/.github/workflows/auto-close-llm-pr.yml +++ b/.github/workflows/auto-close-llm-pr.yml @@ -1,5 +1,7 @@ name: Auto-close LLM PRs -on: +# The workflow only reads the pull request body through the API, it never +# checks out or runs pull request code, so pull_request_target is safe here. +on: # zizmor: ignore[dangerous-triggers] pull_request_target: types: [opened] permissions: @@ -11,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check PR body and close if LLM-written - uses: actions/github-script@v6 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ed2388a59..331fade61 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,4 +1,8 @@ name: Checks + +permissions: + contents: read + on: push: branches: @@ -13,6 +17,10 @@ concurrency: jobs: checks: runs-on: ubuntu-latest + env: + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -38,21 +46,31 @@ jobs: TOXENV: twinecheck steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + docs/requirements.txt + pyproject.toml + tox.ini + - name: Run check env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - uses: pre-commit/action@v3.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 6930c4c1c..82b16eea2 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -22,24 +22,38 @@ permissions: {} jobs: benchmark: runs-on: ubuntu-latest + env: + # Make uv use the interpreter that actions/setup-python installed + # instead of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system permissions: contents: read id-token: write # OIDC authentication with CodSpeed steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python 3.14 - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.14' + + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + - name: Install dependencies + # tox must stay on PATH for the CodSpeed action to invoke it. run: | - pip install --upgrade pip - pip install --upgrade tox + uv tool install --with tox-uv tox tox -n -e benchmark - name: Run benchmarks - uses: CodSpeedHQ/action@v4 + uses: CodSpeedHQ/action@f22792bfac16f3e14eb9fbea76f4a48e9cc22b93 # v4.19.1 with: mode: simulation run: tox -e benchmark diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7779bbb6b..697647131 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,8 @@ name: Publish + +permissions: + contents: read + on: push: tags: @@ -9,8 +13,28 @@ concurrency: cancel-in-progress: true jobs: + build: + name: Build distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + - run: | + python -m pip install --upgrade build + python -m build + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-package-distributions + path: dist/ + publish: name: Upload release to PyPI + needs: + - build runs-on: ubuntu-latest environment: name: pypi @@ -18,12 +42,9 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - python-version: "3.14" - - run: | - python -m pip install --upgrade build - python -m build + name: python-package-distributions + path: dist/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 566b34e50..af2a0206a 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,4 +1,8 @@ name: macOS + +permissions: + contents: read + on: push: branches: @@ -15,6 +19,9 @@ jobs: runs-on: macos-latest env: PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -31,25 +38,39 @@ jobs: TOXENV: no-reactor steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + run: uv tool install mitmproxy + - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox - name: Upload coverage report if: ${{ matrix.coverage }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: report_type: test_results diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index ad2bcfce7..60a2bec21 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,4 +1,8 @@ name: Ubuntu + +permissions: + contents: read + on: push: branches: @@ -15,6 +19,9 @@ jobs: runs-on: ubuntu-latest env: PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -92,10 +99,12 @@ jobs: coverage: true steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} @@ -105,21 +114,32 @@ jobs: sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + - name: Install mitmproxy - run: pipx install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + # mitmproxy has no PyPy wheels, so run it on CPython regardless of the + # interpreter under test. + run: uv tool install --python cpython mitmproxy - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox - name: Upload coverage report if: ${{ matrix.coverage }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: report_type: test_results diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index c33f96b12..884ff8e7c 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,4 +1,8 @@ name: Windows + +permissions: + contents: read + on: push: branches: @@ -15,6 +19,9 @@ jobs: runs-on: windows-latest env: PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -55,25 +62,39 @@ jobs: TOXENV: extra-deps steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + run: uv tool install mitmproxy + - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox - name: Upload coverage report if: ${{ matrix.coverage }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: report_type: test_results diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c27348c7a..c2cb5056d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,11 @@ repos: hooks: - id: sphinx-lint - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.8.9 + rev: 0.8.10 hooks: - id: sphinx-scrapy +- repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.28.0 + hooks: + - id: zizmor + args: [--no-progress, --fix] diff --git a/docs/requirements.in b/docs/requirements.in index 257365380..3783dd1dc 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -5,4 +5,4 @@ sphinx sphinx-notfound-page sphinx-rtd-theme sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10 diff --git a/docs/requirements.txt b/docs/requirements.txt index 87634cea9..0f5969401 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0 # via # -r docs/requirements.in # sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@912ed0507405e16ac60a47dd08195a1cd0ced984 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8 # via -r docs/requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy diff --git a/tox.ini b/tox.ini index 8331e2ab6..edde83356 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,8 @@ [tox] requires = - sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9 + sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10 + tox-uv envlist = pre-commit pylint From 11d1712a2cdfde86be037553f18780c45d534222 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 1 Aug 2026 12:51:45 +0500 Subject: [PATCH 041/111] Force CI job names to not include "true" for "coverage". (#7848) --- .github/workflows/tests-macos.yml | 1 + .github/workflows/tests-ubuntu.yml | 1 + .github/workflows/tests-windows.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index af2a0206a..7e928cd76 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -16,6 +16,7 @@ concurrency: jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: macos-latest env: PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 60a2bec21..f929be829 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -16,6 +16,7 @@ concurrency: jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: ubuntu-latest env: PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 884ff8e7c..5d1b1d818 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -16,6 +16,7 @@ concurrency: jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: windows-latest env: PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} From a499dc9511005e026860c5ce72296ba1ff3aae1b Mon Sep 17 00:00:00 2001 From: Adrian Date: Sat, 1 Aug 2026 17:30:50 +0200 Subject: [PATCH 042/111] CI: Reduce the test job matrix while maintaining coverage (#7844) --- .github/workflows/tests-macos.yml | 20 ++++++++++---------- .github/workflows/tests-ubuntu.yml | 8 -------- .github/workflows/tests-windows.yml | 12 ------------ 3 files changed, 10 insertions(+), 30 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 7e928cd76..9a09aa67d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -26,17 +26,17 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] - env: - - TOXENV: py include: - - python-version: '3.14' - env: - TOXENV: py - coverage: true - - python-version: '3.14' - env: - TOXENV: no-reactor + - python-version: "3.10" + env: + TOXENV: py + - python-version: "3.14" + env: + TOXENV: py + coverage: true + - python-version: "3.14" + env: + TOXENV: no-reactor steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f929be829..cd726a2fe 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -51,10 +51,6 @@ jobs: env: TOXENV: no-reactor coverage: true - # pinned due to https://github.com/pypy/pypy/issues/5388 - - python-version: pypy3.11-7.3.20 - env: - TOXENV: pypy3 # min deps - python-version: "3.10.19" @@ -65,10 +61,6 @@ jobs: env: TOXENV: min-default-reactor coverage: true - - python-version: "3.10.19" - env: - TOXENV: min-no-reactor - coverage: true # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 5d1b1d818..254e9395e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -30,22 +30,10 @@ jobs: - python-version: "3.10" env: TOXENV: py - - python-version: "3.11" - env: - TOXENV: py - - python-version: "3.12" - env: - TOXENV: py - - python-version: "3.13" - env: - TOXENV: py - python-version: "3.14" env: TOXENV: py coverage: true - - python-version: "3.14" - env: - TOXENV: default-reactor - python-version: "3.14" env: TOXENV: no-reactor From e83c709574addde91240a3b6bb7eee26c4fd8b32 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 3 Aug 2026 12:44:48 +0200 Subject: [PATCH 043/111] Docs: a job directory belongs to one Scrapy version (#7861) --- docs/topics/jobs.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index dcff10772..c3043204b 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -83,6 +83,14 @@ stopping it cleanly. Forced, sudden or otherwise unclean shutdown can lead to data corruption in the job directory, which may prevent the spider from resuming correctly. +Scrapy version changes +---------------------- + +The contents of a job directory are an implementation detail of the Scrapy +version that wrote them. A job must be resumed with the same Scrapy version +that paused it; after upgrading or downgrading Scrapy, start a new job with a +new job directory. + Cookies expiration ------------------ From 2b2e18199b0bbae9dfe64a111d7fe37de7b7da9a Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 3 Aug 2026 12:49:15 +0200 Subject: [PATCH 044/111] Type downloader middleware tests (#7858) --- pyproject.toml | 12 ---- scrapy/http/request/__init__.py | 4 +- tests/test_downloadermiddleware_cookies.py | 66 +++++++++-------- tests/test_downloadermiddleware_httpauth.py | 8 ++- tests/test_downloadermiddleware_httpcache.py | 31 ++++---- ...st_downloadermiddleware_httpcompression.py | 58 +++++++++++---- tests/test_downloadermiddleware_httpproxy.py | 6 +- tests/test_downloadermiddleware_offsite.py | 20 +++--- tests/test_downloadermiddleware_redirect.py | 20 ++++-- ...wnloadermiddleware_redirect_metarefresh.py | 16 +++-- tests/test_downloadermiddleware_retry.py | 69 +++++++++--------- tests/test_downloadermiddleware_robotstxt.py | 32 ++++----- tests/test_robotstxt_interface.py | 72 ++++++++++--------- 13 files changed, 236 insertions(+), 178 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1cbd39946..11e971a35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,17 +125,6 @@ module = [ "tests.test_contracts", "tests.test_core_downloader", "tests.test_downloader_handler_twisted_ftp", - "tests.test_downloadermiddleware_cookies", - "tests.test_downloadermiddleware_httpauth", - "tests.test_downloadermiddleware_httpcache", - "tests.test_downloadermiddleware_httpcompression", - "tests.test_downloadermiddleware_httpproxy", - "tests.test_downloadermiddleware_offsite", - "tests.test_downloadermiddleware_redirect", - "tests.test_downloadermiddleware_redirect_base", - "tests.test_downloadermiddleware_redirect_metarefresh", - "tests.test_downloadermiddleware_retry", - "tests.test_downloadermiddleware_robotstxt", "tests.test_downloaderslotssettings", "tests.test_dupefilters", "tests.test_engine_loop", @@ -167,7 +156,6 @@ module = [ "tests.test_request_cb_kwargs", "tests.test_request_dict", "tests.test_request_left", - "tests.test_robotstxt_interface", "tests.test_scheduler_base", "tests.test_settings", "tests.test_spider", diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 7d67bb6d7..7c53b6b48 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -50,7 +50,9 @@ class VerboseCookie(TypedDict): secure: NotRequired[bool] -CookiesT: TypeAlias = dict[str | bytes, str | bytes] | list[VerboseCookie] +CookiesT: TypeAlias = ( + dict[str | bytes, str | bytes | bool | float | int] | list[VerboseCookie] +) RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 7ad103f27..8d999d952 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,5 +1,6 @@ import logging from collections.abc import Iterable +from typing import Any import pytest @@ -219,7 +220,7 @@ class TestCookiesMiddleware: def test_complex_cookies(self): # merge some cookies into jar - cookies = [ + cookies: list[VerboseCookie] = [ { "name": "C1", "value": "value1", @@ -483,13 +484,13 @@ class TestCookiesMiddleware: def _test_cookie_redirect( self, - source, - target, + source: str | dict[str, Any], + target: str | dict[str, Any], *, - cookies1, - cookies2, - ): - input_cookies = {"a": "b"} + cookies1: bool, + cookies2: bool, + ) -> None: + input_cookies: CookiesT = {"a": "b"} if not isinstance(source, dict): source = {"url": source} @@ -551,11 +552,11 @@ class TestCookiesMiddleware: def _test_cookie_header_redirect( self, - source, - target, + source: str | dict[str, Any], + target: str | dict[str, Any], *, - cookies2, - ): + cookies2: bool, + ) -> None: """Test the handling of a user-defined Cookie header when building a redirect follow-up request. @@ -623,14 +624,14 @@ class TestCookiesMiddleware: def _test_user_set_cookie_domain_followup( self, - url1, - url2, - domain, + url1: str, + url2: str, + domain: str, *, - cookies1, - cookies2, - ): - input_cookies = [ + cookies1: bool, + cookies2: bool, + ) -> None: + input_cookies: list[VerboseCookie] = [ { "name": "a", "value": "b", @@ -686,16 +687,16 @@ class TestCookiesMiddleware: def _test_server_set_cookie_domain_followup( self, - url1, - url2, - domain, + url1: str, + url2: str, + domain: str, *, - cookies, - ): + cookies: bool, + ) -> None: request1 = Request(url1) self.mw.process_request(request1) - input_cookies = [ + input_cookies: list[VerboseCookie] = [ { "name": "a", "value": "b", @@ -747,8 +748,14 @@ class TestCookiesMiddleware: ) def _test_cookie_redirect_scheme_change( - self, secure, from_scheme, to_scheme, cookies1, cookies2, cookies3 - ): + self, + secure: bool | object, + from_scheme: str, + to_scheme: str, + cookies1: bool, + cookies2: bool, + cookies3: bool, + ) -> None: """When a redirect causes the URL scheme to change from *from_scheme* to *to_scheme*, while domain and port remain the same, and given a cookie on the initial request with its secure attribute set to @@ -756,10 +763,11 @@ class TestCookiesMiddleware: initial request (*cookies1*), if it should be kept by the redirect middleware (*cookies2*), and if it should be present on the Cookie header in the redirected request (*cookie3*).""" - cookie_kwargs = {} + cookie: VerboseCookie = {"name": "a", "value": "b"} if secure is not UNSET: - cookie_kwargs["secure"] = secure - input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}] + assert isinstance(secure, bool) + cookie["secure"] = secure + input_cookies = [cookie] request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies) self.mw.process_request(request1) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 827133d2e..dd5af3bc5 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,3 +1,5 @@ +from typing import Any + import pytest from w3lib.http import basic_auth_header @@ -10,8 +12,10 @@ from scrapy.utils.test import get_crawler _DOMAIN_NOT_SET = object() -def make_mw(user="", passwd="", domain=_DOMAIN_NOT_SET): - settings: dict = { +def make_mw( + user: str = "", passwd: str = "", domain: str | object = _DOMAIN_NOT_SET +) -> HttpAuthMiddleware: + settings: dict[str, Any] = { "HTTPAUTH_USER": user, "HTTPAUTH_PASS": passwd, } diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 6e8486eb8..dc8228470 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -94,14 +94,14 @@ class TestBase: finally: mw.spider_closed(crawler.spider) - def assertEqualResponse(self, response1, response2): + def assertEqualResponse(self, response1: Response, response2: Response) -> None: assert response1.url == response2.url assert response1.status == response2.status assert response1.headers == response2.headers assert response1.body == response2.body -class StorageTestMixin: +class StorageTestMixin(TestBase): """Mixin containing storage-specific test methods.""" def _corrupt_cache_entry( @@ -135,6 +135,8 @@ class StorageTestMixin: def test_corrupted_cache_entry_is_a_miss(self, caplog): with self._middleware() as mw: spider = mw.crawler.spider + assert spider + assert mw.crawler.stats mw.storage.store_response(spider, self.request, self.response) self._corrupt_cache_entry(mw.storage, spider, self.request) @@ -155,6 +157,8 @@ class StorageTestMixin: def test_corrupted_cache_entry_ignore_missing(self): with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: spider = mw.crawler.spider + assert spider + assert mw.crawler.stats mw.storage.store_response(spider, self.request, self.response) self._corrupt_cache_entry(mw.storage, spider, self.request) @@ -180,7 +184,7 @@ class StorageTestMixin: self.assertEqualResponse(response, cached_response) -class PolicyTestMixin: +class PolicyTestMixin(TestBase): """Mixin containing policy-specific test methods.""" def test_dont_cache(self): @@ -302,6 +306,7 @@ class DummyPolicyTestMixin(PolicyTestMixin): assert mw.process_request(self.request) is None fresh_response = self.response.replace(body=b"new body") response = mw.process_response(self.request, fresh_response) + assert isinstance(response, Response) self.assertEqualResponse(self.response, response) assert "cached" in response.flags assert mw.stats.get_value("httpcache/revalidate") == 1 @@ -313,12 +318,12 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): @staticmethod def _process_requestresponse( mw: HttpCacheMiddleware, request: Request, response: Response | None - ) -> Response | Request: - result = None + ) -> Response: + result: Request | Response | None = None try: result = mw.process_request(request) if result: - assert isinstance(result, (Request, Response)) + assert isinstance(result, Response) return result assert response is not None result = mw.process_response(request, response) @@ -346,6 +351,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): res2 = self._process_requestresponse(mw, req0, res0) assert "cached" not in res2.flags res3 = mw.process_request(req0) + assert isinstance(res3, Response) assert "cached" in res3.flags self.assertEqualResponse(res2, res3) # request with no-cache directive must not return cached response @@ -634,6 +640,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): assert mw.process_request(req0) is None res1 = mw.process_exception(req0, e("foo")) # Use cached response as recovery + assert isinstance(res1, Response) assert "cached" in res1.flags self.assertEqualResponse(res0, res1) # Do not use cached response for unhandled exceptions @@ -684,26 +691,22 @@ class DbmStorageTestMixin(StorageTestMixin): class TestFilesystemStorageWithDummyPolicy( - TestBase, FilesystemStorageTestMixin, DummyPolicyTestMixin + FilesystemStorageTestMixin, DummyPolicyTestMixin ): policy_class = "scrapy.extensions.httpcache.DummyPolicy" class TestFilesystemStorageWithRFC2616Policy( - TestBase, FilesystemStorageTestMixin, RFC2616PolicyTestMixin + FilesystemStorageTestMixin, RFC2616PolicyTestMixin ): policy_class = "scrapy.extensions.httpcache.RFC2616Policy" -class TestDbmStorageWithDummyPolicy( - TestBase, DbmStorageTestMixin, DummyPolicyTestMixin -): +class TestDbmStorageWithDummyPolicy(DbmStorageTestMixin, DummyPolicyTestMixin): policy_class = "scrapy.extensions.httpcache.DummyPolicy" -class TestDbmStorageWithRFC2616Policy( - TestBase, DbmStorageTestMixin, RFC2616PolicyTestMixin -): +class TestDbmStorageWithRFC2616Policy(DbmStorageTestMixin, RFC2616PolicyTestMixin): policy_class = "scrapy.extensions.httpcache.RFC2616Policy" diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 55f06b396..fa0707491 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -3,6 +3,7 @@ from importlib.util import find_spec from io import BytesIO from logging import WARNING from pathlib import Path +from typing import Any import pytest from w3lib.encoding import resolve_encoding @@ -15,6 +16,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWar from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider +from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir @@ -72,6 +74,7 @@ class TestHttpCompression: def setup_method(self): self.crawler = get_crawler(Spider) self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) + assert self.crawler.stats self.crawler.stats.open_spider() def _getresponse(self, coding: str) -> Response: @@ -96,7 +99,8 @@ class TestHttpCompression: ) return response - def assertStatsEqual(self, key, value): + def assertStatsEqual(self, key: str, value: Any) -> None: + assert self.crawler.stats assert self.crawler.stats.get_value(key) == value, str( self.crawler.stats.get_stats() ) @@ -145,6 +149,7 @@ class TestHttpCompression: def test_process_response_gzip(self): response = self._getresponse("gzip") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"gzip" @@ -159,6 +164,7 @@ class TestHttpCompression: _skip_if_no_br() response = self._getresponse("br") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"br" newresponse = self.mw.process_response(request, response) @@ -172,6 +178,7 @@ class TestHttpCompression: if find_spec("brotli") is not None or find_spec("brotlicffi") is not None: pytest.skip("Requires not having brotli support") response = self._getresponse("br") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"br" caplog.clear() @@ -201,6 +208,7 @@ class TestHttpCompression: if not check_key.startswith("zstd-"): continue response = self._getresponse(check_key) + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"zstd" newresponse = self.mw.process_response(request, response) @@ -216,6 +224,7 @@ class TestHttpCompression: if find_spec("zstandard") is not None: pytest.skip("Requires not having zstandard support") response = self._getresponse("zstd-static-content-size") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"zstd" caplog.clear() @@ -239,6 +248,7 @@ class TestHttpCompression: def test_process_response_rawdeflate(self): response = self._getresponse("rawdeflate") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"deflate" @@ -251,6 +261,7 @@ class TestHttpCompression: def test_process_response_zlibdelate(self): response = self._getresponse("zlibdeflate") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"deflate" @@ -275,6 +286,7 @@ class TestHttpCompression: def test_multipleencodings(self): response = self._getresponse("gzip") response.headers["Content-Encoding"] = ["uuencode", "gzip"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -282,6 +294,7 @@ class TestHttpCompression: def test_multi_compression_single_header(self): response = self._getresponse("gzip-deflate") + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -293,6 +306,7 @@ class TestHttpCompression: ) -> None: response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = [b"gzip, foo, deflate"] + assert response.request request = response.request caplog.clear() with caplog.at_level( @@ -315,6 +329,7 @@ class TestHttpCompression: def test_multi_compression_multiple_header(self): response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = ["gzip", "deflate"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -324,6 +339,7 @@ class TestHttpCompression: def test_multi_compression_multiple_header_invalid_compression(self): response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -332,6 +348,7 @@ class TestHttpCompression: def test_multi_compression_single_and_multiple_header(self): response = self._getresponse("gzip-deflate-gzip") response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -341,6 +358,7 @@ class TestHttpCompression: def test_multi_compression_single_and_multiple_header_invalid_compression(self): response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = ["gzip", "foo,deflate"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -397,9 +415,7 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_no_content_type_header(self): - headers = { - "Content-Encoding": "identity", - } + headers = {b"Content-Encoding": b"identity"} plainbody = ( b"Some page" b'' @@ -414,6 +430,7 @@ class TestHttpCompression: newresponse = self.mw.process_response(request, response) assert isinstance(newresponse, respcls) + assert isinstance(newresponse, HtmlResponse) assert newresponse.body == plainbody assert newresponse.encoding == resolve_encoding("gb2312") self.assertStatsEqual("httpcompression/response_count", 1) @@ -422,6 +439,7 @@ class TestHttpCompression: def test_process_response_gzipped_contenttype(self): response = self._getresponse("gzip") response.headers["Content-Type"] = "application/gzip" + assert response.request request = response.request newresponse = self.mw.process_response(request, response) @@ -434,6 +452,7 @@ class TestHttpCompression: def test_process_response_gzip_app_octetstream_contenttype(self): response = self._getresponse("gzip") response.headers["Content-Type"] = "application/octet-stream" + assert response.request request = response.request newresponse = self.mw.process_response(request, response) @@ -446,6 +465,7 @@ class TestHttpCompression: def test_process_response_gzip_binary_octetstream_contenttype(self): response = self._getresponse("x-gzip") response.headers["Content-Type"] = "binary/octet-stream" + assert response.request request = response.request newresponse = self.mw.process_response(request, response) @@ -504,6 +524,7 @@ class TestHttpCompression: def test_process_response_head_request_no_decode_required(self): response = self._getresponse("gzip") response.headers["Content-Type"] = "application/gzip" + assert response.request request = response.request request.method = "HEAD" response = response.replace(body=None) @@ -513,7 +534,7 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_bytes", None) - def _test_compression_bomb_setting(self, compression_id): + def _test_compression_bomb_setting(self, compression_id: str) -> None: settings = {"DOWNLOAD_MAXSIZE": 1_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") @@ -521,9 +542,12 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B + assert response.request with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 + cause = exc_info.value.__cause__ + assert isinstance(cause, _DecompressionMaxSizeExceeded) + assert cause.decompressed_size < 1_100_000 def test_compression_bomb_setting_br(self): _skip_if_no_br() @@ -549,6 +573,7 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse("bomb-gzip") # 11_511_612 B + assert response.request caplog.clear() with ( caplog.at_level( @@ -565,7 +590,7 @@ class TestHttpCompression: ) ] - def _test_compression_bomb_spider_attr(self, compression_id): + def _test_compression_bomb_spider_attr(self, compression_id: str) -> None: class DownloadMaxSizeSpider(Spider): download_maxsize = 1_000_000 @@ -575,9 +600,12 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") + assert response.request with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 + cause = exc_info.value.__cause__ + assert isinstance(cause, _DecompressionMaxSizeExceeded) + assert cause.decompressed_size < 1_100_000 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_compression_bomb_spider_attr_br(self): @@ -599,7 +627,7 @@ class TestHttpCompression: self._test_compression_bomb_spider_attr("zstd") - def _test_compression_bomb_request_meta(self, compression_id): + def _test_compression_bomb_request_meta(self, compression_id: str) -> None: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") mw = HttpCompressionMiddleware.from_crawler(crawler) @@ -607,9 +635,12 @@ class TestHttpCompression: response = self._getresponse(f"bomb-{compression_id}") response.meta["download_maxsize"] = 1_000_000 + assert response.request with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 + cause = exc_info.value.__cause__ + assert isinstance(cause, _DecompressionMaxSizeExceeded) + assert cause.decompressed_size < 1_100_000 def test_compression_bomb_request_meta_br(self): _skip_if_no_br() @@ -789,7 +820,7 @@ class TestHttpCompression: self._test_download_warnsize_request_meta(caplog, "zstd") - def _get_truncated_response(self, compression_id): + def _get_truncated_response(self, compression_id: str) -> Response: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") mw = HttpCompressionMiddleware.from_crawler(crawler) @@ -797,7 +828,10 @@ class TestHttpCompression: response = self._getresponse(compression_id) truncated_body = response.body[: len(response.body) // 2] response = response.replace(body=truncated_body) - return mw.process_response(response.request, response) + assert response.request + new_response = mw.process_response(response.request, response) + assert isinstance(new_response, Response) + return new_response def test_process_truncated_response_br(self): _skip_if_no_br() diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 7ed848764..54d4601a7 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -14,7 +14,8 @@ class TestHttpProxyMiddleware: self._oldenv = os.environ.copy() def teardown_method(self): - os.environ = self._oldenv + os.environ.clear() + os.environ.update(self._oldenv) def test_not_enabled(self): crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False}) @@ -22,7 +23,8 @@ class TestHttpProxyMiddleware: HttpProxyMiddleware.from_crawler(crawler) def test_no_environment_proxies(self): - os.environ = {"dummy_proxy": "reset_env_and_do_not_raise"} + os.environ.clear() + os.environ["dummy_proxy"] = "reset_env_and_do_not_raise" mw = HttpProxyMiddleware() for url in ("http://e.com", "https://e.com", "file:///tmp/a"): diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index 78efb0191..cb17c2553 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -1,4 +1,5 @@ import re +from typing import Any import pytest @@ -53,7 +54,7 @@ def test_process_request_dont_filter(value, filtered): crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["dont_filter"] = value request = Request("https://b.example", **kwargs) @@ -82,7 +83,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) - kwargs = {"meta": {}} + kwargs: dict[str, Any] = {"meta": {}} if allow_offsite is not UNSET: kwargs["meta"]["allow_offsite"] = allow_offsite if dont_filter is not UNSET: @@ -105,7 +106,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): ) def test_process_request_no_allowed_domains(value): crawler = get_crawler(Spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["allowed_domains"] = value crawler.spider = crawler._create_spider(name="a", **kwargs) @@ -152,7 +153,7 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): mw.spider_opened(crawler.spider) request = Request(url) if allowed: - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) else: with pytest.raises(IgnoreRequest): mw.request_scheduled(request, crawler.spider) @@ -172,7 +173,7 @@ def test_request_scheduled_dont_filter(value, filtered): crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["dont_filter"] = value request = Request("https://b.example", **kwargs) @@ -180,7 +181,7 @@ def test_request_scheduled_dont_filter(value, filtered): with pytest.raises(IgnoreRequest): mw.request_scheduled(request, crawler.spider) else: - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) @pytest.mark.parametrize( @@ -193,14 +194,14 @@ def test_request_scheduled_dont_filter(value, filtered): ) def test_request_scheduled_no_allowed_domains(value): crawler = get_crawler(Spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["allowed_domains"] = value crawler.spider = crawler._create_spider(name="a", **kwargs) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) request = Request("https://example.com") - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) def test_request_scheduled_invalid_domains(): @@ -210,7 +211,7 @@ def test_request_scheduled_invalid_domains(): mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) request = Request("https://a.example") - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) for letter in ("b", "c"): request = Request(f"https://{letter}.example") with pytest.raises(IgnoreRequest): @@ -227,6 +228,7 @@ def test_repeated_offsite_domain(): with pytest.raises(IgnoreRequest): mw.process_request(req1) assert "other.org" in mw.domains_seen + assert crawler.stats assert crawler.stats.get_value("offsite/domains") == 1 assert crawler.stats.get_value("offsite/filtered") == 1 with pytest.raises(IgnoreRequest): diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index ef2774a93..de97aaadb 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -309,7 +309,7 @@ class TestRedirectMiddleware(TestRedirectBase): url = "http://www.example.com/301" url2 = "http://www.example.com/redirected" - def _test_passthrough(req): + def _test_passthrough(req: Request) -> None: rsp = Response(url, headers={"Location": url2}, status=301, request=req) r = self.mw.process_response(req, rsp) assert r is rsp @@ -404,15 +404,17 @@ def test_response_referrer_policy(policy, source_url, target_url, expected_refer status=301, headers={"Location": target_url, **extra_headers}, ) - source_request = redirect_mw.process_response(source_request, response_redirect) - assert isinstance(source_request, Request) + target_request = redirect_mw.process_response(source_request, response_redirect) + assert isinstance(target_request, Request) - assert source_request.headers.get("Referer") == expected_referrer + assert target_request.headers.get("Referer") == expected_referrer def test_no_warning_when_referer_middleware_present(caplog): crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=MagicMock()) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=MagicMock() + ) mw = build_from_crawler(RedirectMiddleware, crawler) caplog.clear() with caplog.at_level(logging.WARNING): @@ -426,7 +428,9 @@ def test_no_warning_when_referer_middleware_present(caplog): def test_warning_redirect_middleware(caplog): crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=None) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=None + ) mw = build_from_crawler(RedirectMiddleware, crawler) with caplog.at_level(logging.WARNING): mw._engine_started() @@ -449,7 +453,9 @@ def test_warning_subclass(caplog): pass crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=None) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=None + ) mw = build_from_crawler(MyRedirectMiddleware, crawler) with caplog.at_level(logging.WARNING): mw._engine_started() diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py index aeae759a0..83dc6825f 100644 --- a/tests/test_downloadermiddleware_redirect_metarefresh.py +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -21,7 +21,7 @@ from tests.utils.redirect import ( ) -def meta_refresh_body(url, interval=5): +def meta_refresh_body(url: str, interval: int = 5) -> bytes: html = f"""""" return html.encode("utf-8") @@ -34,10 +34,14 @@ class TestMetaRefreshMiddleware(TestRedirectBase): crawler = get_crawler(Spider) self.mw = self.mwcls.from_crawler(crawler) - def _body(self, interval=5, url="http://example.org/newpage"): + def _body( + self, interval: int = 5, url: str = "http://example.org/newpage" + ) -> bytes: return meta_refresh_body(url, interval) - def get_response(self, request, location): + def get_response( + self, request: Request, location: str, status: int = 302 + ) -> Response: return HtmlResponse(request.url, body=self._body(url=location)) def test_meta_refresh(self): @@ -75,7 +79,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase): assert "Content-Length" not in req2.headers, ( "Content-Length header must not be present in redirected request" ) - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" + assert not req2.body, f"Redirected body must be empty, not {req2.body!r}" def test_ignore_tags_default(self): req = Request(url="http://example.org") @@ -142,7 +146,9 @@ def test_meta_refresh_schemes(url, location, target): def test_warning_meta_refresh_middleware(caplog): crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=None) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=None + ) mw = build_from_crawler(MetaRefreshMiddleware, crawler) with caplog.at_level(logging.WARNING): mw._engine_started() diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 410427b84..ab52590c7 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -31,6 +31,7 @@ class TestRetry: req = Request("http://www.scrapytest.org/503") rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) assert req2.priority < req.priority def test_404(self): @@ -53,9 +54,9 @@ class TestRetry: rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 1 + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.meta["retry_times"] == 1 def test_dont_retry_exc(self): req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) @@ -68,18 +69,19 @@ class TestRetry: rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 1 + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.meta["retry_times"] == 1 # second retry - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 2 + req3 = self.mw.process_response(req2, rsp) + assert isinstance(req3, Request) + assert req3.meta["retry_times"] == 2 # discard it - assert self.mw.process_response(req, rsp) is rsp + assert self.mw.process_response(req3, rsp) is rsp + assert self.crawler.stats assert self.crawler.stats.get_value("retry/max_reached") == 1 assert ( self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable") @@ -131,6 +133,7 @@ class TestRetry: self._test_retry_exception(req, exc("foo")) stats = self.crawler.stats + assert stats assert stats.get_value("retry/max_reached") == len(exceptions) assert stats.get_value("retry/count") == len(exceptions) * 2 assert ( @@ -149,29 +152,30 @@ class TestRetry: req = Request(f"http://www.scrapytest.org/{exc.__name__}") self._test_retry_exception(req, exc("foo"), mw) - def _test_retry_exception(self, req, exception, mw=None): + def _test_retry_exception( + self, req: Request, exception: Exception, mw: RetryMiddleware | None = None + ) -> None: if mw is None: mw = self.mw # first retry - req = mw.process_exception(req, exception) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 1 + req2 = mw.process_exception(req, exception) + assert isinstance(req2, Request) + assert req2.meta["retry_times"] == 1 # second retry - req = mw.process_exception(req, exception) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 2 + req3 = mw.process_exception(req2, exception) + assert isinstance(req3, Request) + assert req3.meta["retry_times"] == 2 # discard it - req = mw.process_exception(req, exception) - assert req is None + assert mw.process_exception(req3, exception) is None class TestMaxRetryTimes: invalid_url = "http://www.scrapytest.org/invalid_url" - def get_middleware(self, settings=None): + def get_middleware(self, settings: dict[str, Any] | None = None) -> RetryMiddleware: crawler = get_crawler(DefaultSpider, settings or {}) crawler.spider = crawler._create_spider() return RetryMiddleware.from_crawler(crawler) @@ -275,20 +279,18 @@ class TestMaxRetryTimes: def _test_retry( self, - req, - exception, - max_retry_times, - middleware=None, - ): - middleware = middleware or self.mw - + req: Request, + exception: Exception, + max_retry_times: int, + middleware: RetryMiddleware, + ) -> None: for _ in range(max_retry_times): - req = middleware.process_exception(req, exception) - assert isinstance(req, Request) + result = middleware.process_exception(req, exception) + assert isinstance(result, Request) + req = result # discard it - req = middleware.process_exception(req, exception) - assert req is None + assert middleware.process_exception(req, exception) is None class TestGetRetryRequest: @@ -428,7 +430,7 @@ class TestGetRetryRequest: def test_no_spider(self): request = Request("https://example.com") with pytest.raises(TypeError): - get_retry_request(request) # pylint: disable=missing-kwoa + get_retry_request(request) # type: ignore[call-arg] # pylint: disable=missing-kwoa def test_max_retry_times_setting(self): max_retry_times = 0 @@ -471,6 +473,7 @@ class TestGetRetryRequest: request, spider=spider, ) + assert new_request assert new_request.priority == priority_adjust def test_priority_adjust_argument(self): @@ -482,6 +485,7 @@ class TestGetRetryRequest: spider=spider, priority_adjust=priority_adjust, ) + assert new_request assert new_request.priority == priority_adjust def test_log_extra_retry_success(self, caplog: pytest.LogCaptureFixture) -> None: @@ -732,6 +736,7 @@ class TestGetRetryRequest: reason=expected_reason, stats_base_key=stats_key, ) + assert spider.crawler.stats for stat in ( f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}", diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 1f2575f8f..793a2b5be 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -from typing import TYPE_CHECKING from unittest import mock import pytest @@ -19,9 +18,6 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from tests.utils.decorators import coroutine_test from tests.utils.robotstxt import rerp_available -if TYPE_CHECKING: - from scrapy.crawler import Crawler - class TestRobotsTxtMiddleware: def setup_method(self) -> None: @@ -39,7 +35,7 @@ class TestRobotsTxtMiddleware: with pytest.raises(NotConfigured): RobotsTxtMiddleware(self.crawler) - def _get_successful_crawler(self) -> Crawler: + def _get_successful_crawler(self) -> mock.MagicMock: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) ROBOTS = """ @@ -54,8 +50,8 @@ Disallow: /some/randome/page.html """.encode() response = TextResponse("http://site.local/robots.txt", body=ROBOTS) - async def return_response(request): - deferred = Deferred() + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.callback, response) return await maybe_deferred_to_future(deferred) @@ -130,15 +126,15 @@ Disallow: /some/randome/page.html Request("http://site.local/static/", meta=meta), middleware ) - def _get_garbage_crawler(self) -> Crawler: + def _get_garbage_crawler(self) -> mock.MagicMock: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response( "http://site.local/robots.txt", body=b"GIF89a\xd3\x00\xfe\x00\xa2" ) - async def return_response(request): - deferred = Deferred() + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.callback, response) return await maybe_deferred_to_future(deferred) @@ -154,13 +150,13 @@ Disallow: /some/randome/page.html await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) await self.assertNotIgnored(Request("http://site.local/static/"), middleware) - def _get_emptybody_crawler(self) -> Crawler: + def _get_emptybody_crawler(self) -> mock.MagicMock: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response("http://site.local/robots.txt") - async def return_response(request): - deferred = Deferred() + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.callback, response) return await maybe_deferred_to_future(deferred) @@ -180,8 +176,8 @@ Disallow: /some/randome/page.html self.crawler.settings.set("ROBOTSTXT_OBEY", True) err = CannotResolveHostError("Robotstxt address not found") - async def return_failure(request): - deferred = Deferred() + async def return_failure(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.errback, failure.Failure(err)) return await maybe_deferred_to_future(deferred) @@ -208,8 +204,8 @@ Disallow: /some/randome/page.html async def test_ignore_robotstxt_request(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) - async def ignore_request(request): - deferred = Deferred() + async def ignore_request(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.errback, failure.Failure(IgnoreRequest())) return await maybe_deferred_to_future(deferred) @@ -236,7 +232,7 @@ Disallow: /some/randome/page.html @coroutine_test async def test_robotstxt_local_file(self): middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) - middleware.process_request_2 = mock.MagicMock() + middleware.process_request_2 = mock.MagicMock() # type: ignore[method-assign] await middleware.process_request(Request("data:text/plain,Hello World data")) assert not middleware.process_request_2.called diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index ea67877f8..755f29959 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,3 +1,7 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import pytest from scrapy.robotstxt import ( @@ -10,22 +14,32 @@ from scrapy.robotstxt import ( from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER from tests.utils.robotstxt import rerp_available +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + class BaseRobotParserTest: - def _setUp(self, parser_cls): + parser_cls: type[RobotParser] + + def _setUp(self, parser_cls: type[RobotParser]) -> None: self.parser_cls = parser_cls + def _parse(self, robotstxt_body: bytes) -> RobotParser: + # The parser backends only use the crawler to get the spider to log with. + return self.parser_cls.from_crawler(None, robotstxt_body) # type: ignore[arg-type] + def test_allowed(self): robotstxt_robotstxt_body = ( b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10" ) - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://www.site.local/allowed", "*") assert not rp.allowed("https://www.site.local/disallowed", "*") - def test_allowed_wildcards(self): + def test_allowed_wildcards(self) -> None: robotstxt_robotstxt_body = b"""User-agent: first Disallow: /disallowed/*/end$ @@ -33,9 +47,7 @@ class BaseRobotParserTest: Allow: /*allowed Disallow: / """ - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://www.site.local/disallowed", "first") assert not rp.allowed("https://www.site.local/disallowed/xyz/end", "first") @@ -46,23 +58,19 @@ class BaseRobotParserTest: assert rp.allowed("https://www.site.local/is_still_allowed", "second") assert rp.allowed("https://www.site.local/is_allowed_too", "second") - def test_length_based_precedence(self): + def test_length_based_precedence(self) -> None: robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://www.site.local/page", "*") - def test_order_based_precedence(self): + def test_order_based_precedence(self) -> None: robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert not rp.allowed("https://www.site.local/page", "*") def test_empty_response(self): """empty response should equal 'allow all'""" - rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b"") + rp = self._parse(b"") assert rp.allowed("https://site.local/", "*") assert rp.allowed("https://site.local/", "chrome") assert rp.allowed("https://site.local/index.html", "*") @@ -71,9 +79,7 @@ class BaseRobotParserTest: def test_garbage_response(self): """garbage response should be discarded, equal 'allow all'""" robotstxt_robotstxt_body = b"GIF89a\xd3\x00\xfe\x00\xa2" - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://site.local/", "*") assert rp.allowed("https://site.local/", "chrome") assert rp.allowed("https://site.local/index.html", "*") @@ -81,12 +87,12 @@ class BaseRobotParserTest: def test_crawl_delay(self): robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n" - rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body) + rp = self._parse(robotstxt_body) assert rp.crawl_delay("*") == 10.0 def test_crawl_delay_unset(self): robotstxt_body = b"User-agent: *\nDisallow: /private\n" - rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body) + rp = self._parse(robotstxt_body) assert rp.crawl_delay("*") is None def test_unicode_url_and_useragent(self): @@ -100,9 +106,7 @@ class BaseRobotParserTest: User-Agent: UnicödeBöt Disallow: /some/randome/page.html""".encode() - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://site.local/", "*") assert not rp.allowed("https://site.local/admin/", "*") assert not rp.allowed("https://site.local/static/", "*") @@ -117,15 +121,13 @@ class TestRobotParser: def test_crawl_delay_unsupported(self): class AllowAllRobotParser(RobotParser): @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: return cls() - def allowed(self, url, user_agent): + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: return True - rp = AllowAllRobotParser.from_crawler( - crawler=None, robotstxt_body=b"User-agent: *\nCrawl-delay: 10\n" - ) + rp = AllowAllRobotParser() assert rp.crawl_delay("*") is None @@ -162,21 +164,21 @@ class TestPythonRobotParser(BaseRobotParserTest): not STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support length based directives precedence.", ) - def test_length_based_precedence(self): + def test_length_based_precedence(self) -> None: super().test_length_based_precedence() @pytest.mark.skipif( STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support order based directives precedence.", ) - def test_order_based_precedence(self): + def test_order_based_precedence(self) -> None: super().test_order_based_precedence() @pytest.mark.skipif( not STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support wildcards.", ) - def test_allowed_wildcards(self): + def test_allowed_wildcards(self) -> None: super().test_allowed_wildcards() @@ -185,7 +187,7 @@ class TestRerpRobotParser(BaseRobotParserTest): def setup_method(self): super()._setUp(RerpRobotParser) - def test_length_based_precedence(self): + def test_length_based_precedence(self) -> None: pytest.skip("Rerp does not support length based directives precedence.") @@ -193,5 +195,5 @@ class TestProtegoRobotParser(BaseRobotParserTest): def setup_method(self): super()._setUp(ProtegoRobotParser) - def test_order_based_precedence(self): + def test_order_based_precedence(self) -> None: pytest.skip("Protego does not support order based directives precedence.") From a9f177030685281550481888a33e45cb0fc1c9ee Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 3 Aug 2026 13:03:28 +0200 Subject: [PATCH 045/111] Docs: sort and compact the component-settings list (#7862) --- docs/topics/components.rst | 47 +++++++++++++------------------------- 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/docs/topics/components.rst b/docs/topics/components.rst index c0df86922..354375577 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -9,37 +9,22 @@ A Scrapy component is any class whose objects are built using That includes the classes that you may assign to the following settings: -- :setting:`ADDONS` - -- :setting:`TWISTED_DNS_RESOLVER` - -- :setting:`DOWNLOAD_HANDLERS` - -- :setting:`DOWNLOADER_MIDDLEWARES` - -- :setting:`DUPEFILTER_CLASS` - -- :setting:`EXTENSIONS` - -- :setting:`FEED_EXPORTERS` - -- :setting:`FEED_STORAGES` - -- :setting:`ITEM_PIPELINES` - -- :setting:`SCHEDULER` - -- :setting:`SCHEDULER_DISK_QUEUE` - -- :setting:`SCHEDULER_MEMORY_QUEUE` - -- :setting:`SCHEDULER_PRIORITY_QUEUE` - -- :setting:`SCHEDULER_START_DISK_QUEUE` - -- :setting:`SCHEDULER_START_MEMORY_QUEUE` - -- :setting:`SPIDER_MIDDLEWARES` +- :setting:`ADDONS` +- :setting:`DOWNLOAD_HANDLERS` +- :setting:`DOWNLOADER_MIDDLEWARES` +- :setting:`DUPEFILTER_CLASS` +- :setting:`EXTENSIONS` +- :setting:`FEED_EXPORTERS` +- :setting:`FEED_STORAGES` +- :setting:`ITEM_PIPELINES` +- :setting:`SCHEDULER` +- :setting:`SCHEDULER_DISK_QUEUE` +- :setting:`SCHEDULER_MEMORY_QUEUE` +- :setting:`SCHEDULER_PRIORITY_QUEUE` +- :setting:`SCHEDULER_START_DISK_QUEUE` +- :setting:`SCHEDULER_START_MEMORY_QUEUE` +- :setting:`SPIDER_MIDDLEWARES` +- :setting:`TWISTED_DNS_RESOLVER` Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to From cde7af87fa715ba354be315ca9159c386f1c7774 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 3 Aug 2026 13:55:39 +0200 Subject: [PATCH 046/111] Cover passing spider to a deprecated Downloader.fetch() (#7863) --- tests/test_engine_download.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_engine_download.py b/tests/test_engine_download.py index 962808d96..09b998f6a 100644 --- a/tests/test_engine_download.py +++ b/tests/test_engine_download.py @@ -116,6 +116,18 @@ class TestEngineDownloadAsync: engine._slot.add_request.assert_called_once_with(request) engine._slot.remove_request.assert_called_once_with(request) + @coroutine_test + async def test_download_async_fetch_needs_spider(self, engine): + engine._downloader_fetch_needs_spider = True + request = Request("http://example.com") + response = Response("http://example.com", body=b"test body") + engine.spider = Mock() + engine.downloader.fetch.return_value = defer.succeed(response) + + result = await self._download(engine, request) + assert result == response + engine.downloader.fetch.assert_called_once_with(request, engine.spider) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestEngineDownload(TestEngineDownloadAsync): From 298c9e610ec29dea378a706e925ad0d897007410 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 3 Aug 2026 14:16:23 +0200 Subject: [PATCH 047/111] Type tests related to requests and responses (#7864) --- pyproject.toml | 11 ---- scrapy/http/request/form.py | 8 +-- tests/test_http_headers.py | 29 +++++----- tests/test_http_request.py | 3 +- tests/test_http_request_form.py | 75 +++++++++++++++---------- tests/test_http_response_text.py | 15 +++-- tests/test_request_attribute_binding.py | 5 ++ tests/test_request_cb_kwargs.py | 26 ++++++--- tests/test_request_dict.py | 58 +++++++++++-------- tests/test_request_left.py | 43 +++++++------- tests/utils/bases/http_request.py | 59 ++++++++++--------- tests/utils/bases/http_response.py | 48 ++++++++-------- 12 files changed, 214 insertions(+), 166 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 11e971a35..609c70708 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -136,11 +136,6 @@ module = [ "tests.test_feedexport_storages", "tests.test_feedexport_uri_params", "tests.test_http2_client_protocol", - "tests.test_http_headers", - "tests.test_http_request", - "tests.test_http_request_form", - "tests.test_http_response", - "tests.test_http_response_text", "tests.test_item", "tests.test_linkextractors", "tests.test_loader", @@ -152,10 +147,6 @@ module = [ "tests.test_pipeline_media", "tests.test_pipelines", "tests.test_pqueues", - "tests.test_request_attribute_binding", - "tests.test_request_cb_kwargs", - "tests.test_request_dict", - "tests.test_request_left", "tests.test_scheduler_base", "tests.test_settings", "tests.test_spider", @@ -166,8 +157,6 @@ module = [ "tests.test_squeues", "tests.test_squeues_request", "tests.test_stats", - "tests.utils.bases.http_request", - "tests.utils.bases.http_response", "tests.utils.bases.spider", ] check_untyped_defs = false diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index f1a8dbf3b..12745292b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from warnings import warn @@ -34,7 +34,7 @@ if TYPE_CHECKING: FormdataVType: TypeAlias = str | Iterable[str] FormdataKVType: TypeAlias = tuple[str, FormdataVType] -FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None +FormdataType: TypeAlias = Mapping[str, FormdataVType] | Iterable[FormdataKVType] | None class FormRequest(Request): @@ -100,7 +100,7 @@ class FormRequest(Request): super().__init__(*args, **kwargs) if formdata: - items = formdata.items() if isinstance(formdata, dict) else formdata + items = formdata.items() if isinstance(formdata, Mapping) else formdata form_query_str = _urlencode(items, self.encoding) if self.method == "POST": self.headers.setdefault( @@ -248,7 +248,7 @@ def _get_inputs( if clickable and clickable[0] not in formdata and clickable[0] is not None: values.append(clickable) - formdata_items = formdata.items() if isinstance(formdata, dict) else formdata + formdata_items = formdata.items() if isinstance(formdata, Mapping) else formdata values.extend((k, v) for k, v in formdata_items if v is not None) return values diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index aff3562e3..e7ed17615 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -6,9 +6,6 @@ from scrapy.http import Headers class TestHeaders: - def assertSortedEqual(self, first, second, msg=None): - assert sorted(first) == sorted(second), msg - def test_basics(self): h = Headers({"Content-Type": "text/html", "Content-Length": 1234}) assert h["Content-Type"] @@ -39,7 +36,7 @@ class TestHeaders: assert h["X-Forwarded-For"] == b"ip2" assert h.get("X-Forwarded-For") == b"ip2" assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"] - assert h.getlist("X-Forwarded-For") is not hlist + assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap] def test_multivalue_for_one_header(self): h = Headers((("a", "b"), ("a", "c"))) @@ -49,19 +46,19 @@ class TestHeaders: def test_encode_utf8(self): h = Headers({"key": "\xa3"}, encoding="utf-8") - key, val = dict(h).popitem() + key, val = dict(h.items()).popitem() assert isinstance(key, bytes), key assert isinstance(val[0], bytes), val[0] assert val[0] == b"\xc2\xa3" def test_encode_latin1(self): h = Headers({"key": "\xa3"}, encoding="latin1") - _, val = dict(h).popitem() + _, val = dict(h.items()).popitem() assert val[0] == b"\xa3" def test_encode_multiple(self): h = Headers({"key": ["\xa3"]}, encoding="utf-8") - _, val = dict(h).popitem() + _, val = dict(h.items()).popitem() assert val[0] == b"\xc2\xa3" def test_delete_and_contains(self): @@ -75,7 +72,7 @@ class TestHeaders: h = Headers() hlist = ["ip1", "ip2"] olist = h.setdefault("X-Forwarded-For", hlist) - assert h.getlist("X-Forwarded-For") is not hlist + assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap] assert h.getlist("X-Forwarded-For") is olist h = Headers() @@ -87,16 +84,16 @@ class TestHeaders: idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]} h = Headers(idict) - assert dict(h) == { + assert dict(h.items()) == { b"Content-Type": [b"text/html"], b"X-Forwarded-For": [b"ip1", b"ip2"], } - self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"]) - self.assertSortedEqual( - h.items(), - [(b"X-Forwarded-For", [b"ip1", b"ip2"]), (b"Content-Type", [b"text/html"])], - ) - self.assertSortedEqual(h.values(), [b"ip2", b"text/html"]) + assert sorted(h.keys()) == [b"Content-Type", b"X-Forwarded-For"] + assert sorted(h.items()) == [ + (b"Content-Type", [b"text/html"]), + (b"X-Forwarded-For", [b"ip1", b"ip2"]), + ] + assert set(h.values()) == {b"ip2", b"text/html"} def test_update(self): h = Headers() @@ -162,4 +159,4 @@ class TestHeaders: with pytest.raises(TypeError, match="Unsupported value type"): Headers().setdefault("foo", object()) with pytest.raises(TypeError, match="Unsupported value type"): - Headers().setlist("foo", [object()]) + Headers().setlist("foo", [object()]) # type: ignore[list-item] diff --git a/tests/test_http_request.py b/tests/test_http_request.py index e58ae8f39..b9cec93a1 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,4 +1,5 @@ import xmlrpc.client +from typing import Any import pytest @@ -17,7 +18,7 @@ class TestXmlRpcRequest(TestRequestBase): default_method = "POST" default_headers = {b"Content-Type": [b"text/xml"]} - def _test_request(self, **kwargs): + def _test_request(self, **kwargs: Any) -> None: r = self.request_class("http://scrapytest.org/rpc2", **kwargs) assert r.headers[b"Content-Type"] == b"text/xml" assert r.body == to_bytes( diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py index 5e965e8dc..cb18a0b03 100644 --- a/tests/test_http_request_form.py +++ b/tests/test_http_request_form.py @@ -2,6 +2,7 @@ from __future__ import annotations import re import warnings +from typing import TYPE_CHECKING, Any from urllib.parse import parse_qs, unquote_to_bytes import pytest @@ -12,20 +13,32 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode from tests.utils.bases.http_request import TestRequestBase +if TYPE_CHECKING: + from scrapy import Request -def _buildresponse(body, **kwargs): + +def _buildresponse(body: bytes | str, **kwargs: Any) -> HtmlResponse: kwargs.setdefault("body", body) kwargs.setdefault("url", "http://example.com") kwargs.setdefault("encoding", "utf-8") return HtmlResponse(**kwargs) -def _qs(req, encoding="utf-8", to_unicode=False): - qs = req.body if req.method == "POST" else req.url.partition("?")[2] - uqs = unquote_to_bytes(qs) - if to_unicode: - uqs = uqs.decode(encoding) - return parse_qs(uqs, True) +def _query_string(req: Request) -> bytes: + return req.body if req.method == "POST" else req.url.partition("?")[2].encode() + + +def _qs(req: Request) -> dict[bytes, list[bytes]]: + return parse_qs(unquote_to_bytes(_query_string(req)), True) + + +def _qs_unicode(req: Request, encoding: str = "utf-8") -> dict[str, list[str]]: + qs = unquote_to_bytes(_query_string(req)).decode(encoding) + return parse_qs(qs, True) + + +def _assert_query_equal(first: bytes, second: bytes) -> None: + assert sorted(to_unicode(first).split("&")) == sorted(to_unicode(second).split("&")) # FormRequest.from_response() is deprecated in favor of form2request, so the @@ -34,11 +47,6 @@ def _qs(req, encoding="utf-8", to_unicode=False): class TestFormRequest(TestRequestBase): request_class = FormRequest - def assertQueryEqual(self, first, second, msg=None): - first = to_unicode(first).split("&") - second = to_unicode(second).split("&") - assert sorted(first) == sorted(second), msg - def test_init_not_deprecated(self): # Building a request directly from form data is not deprecated. with warnings.catch_warnings(): @@ -75,20 +83,22 @@ class TestFormRequest(TestRequestBase): assert fs[b"b"] == [b"2"] assert fs.get(b"c") is None - data = {"a": "1", "b": "2"} + mapping = {"a": "1", "b": "2"} fs = _qs( - self.request_class("http://www.example.com/", method="GET", formdata=data) + self.request_class( + "http://www.example.com/", method="GET", formdata=mapping + ) ) assert fs[b"a"] == [b"1"] assert fs[b"b"] == [b"2"] def test_default_encoding_bytes(self): # using default encoding (utf-8) - data = {b"one": b"two", b"price": b"\xc2\xa3 100"} + data: dict[Any, Any] = {b"one": b"two", b"price": b"\xc2\xa3 100"} r2 = self.request_class("http://www.example.com", formdata=data) assert r2.method == "POST" assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") + _assert_query_equal(r2.body, b"price=%C2%A3+100&one=two") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_textual_data(self): @@ -97,26 +107,26 @@ class TestFormRequest(TestRequestBase): r2 = self.request_class("http://www.example.com", formdata=data) assert r2.method == "POST" assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") + _assert_query_equal(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_mixed_data(self): # using default encoding (utf-8) - data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} + data: dict[Any, Any] = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} r2 = self.request_class("http://www.example.com", formdata=data) assert r2.method == "POST" assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") + _assert_query_equal(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_bytes(self): - data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} + data: dict[Any, Any] = {b"\xb5 one": b"two", b"price": b"\xa3 100"} r2 = self.request_class( "http://www.example.com", formdata=data, encoding="latin1" ) assert r2.method == "POST" assert r2.encoding == "latin1" - self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") + _assert_query_equal(r2.body, b"price=%A3+100&%B5+one=two") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_textual_data(self): @@ -131,7 +141,7 @@ class TestFormRequest(TestRequestBase): # using multiples values for a single key data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]} r3 = self.request_class("http://www.example.com", formdata=data) - self.assertQueryEqual( + _assert_query_equal( r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100" ) @@ -173,7 +183,7 @@ class TestFormRequest(TestRequestBase): assert req.method == "POST" assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert set(fs["test £"]) == {"val1", "val2"} assert set(fs["one"]) == {"two", "three"} assert fs["test2"] == ["xxx µ"] @@ -196,7 +206,7 @@ class TestFormRequest(TestRequestBase): assert req.method == "POST" assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True, encoding="latin1") + fs = _qs_unicode(req, encoding="latin1") assert set(fs["test £"]) == {"val1", "val2"} assert set(fs["one"]) == {"two", "three"} assert fs["test2"] == ["xxx µ"] @@ -218,7 +228,7 @@ class TestFormRequest(TestRequestBase): assert req.method == "POST" assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert set(fs["test £"]) == {"val1", "val2"} assert set(fs["one"]) == {"two", "three"} assert fs["test2"] == ["xxx µ"] @@ -305,7 +315,10 @@ class TestFormRequest(TestRequestBase): """ ) - req = self.request_class.from_response(response, formdata={"two": None}) + req = self.request_class.from_response( + response, + formdata={"two": None}, # type: ignore[arg-type] + ) fs = _qs(req) assert fs[b"one"] == [b"1"] assert b"two" not in fs @@ -450,7 +463,7 @@ class TestFormRequest(TestRequestBase): req = self.request_class.from_response( response, clickdata={"name": "price in \u00a3"} ) - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert fs["price in \u00a3"] def test_from_response_unicode_clickdata_latin1(self): @@ -466,7 +479,7 @@ class TestFormRequest(TestRequestBase): req = self.request_class.from_response( response, clickdata={"name": "price in \u00a5"} ) - fs = _qs(req, to_unicode=True, encoding="latin1") + fs = _qs_unicode(req, encoding="latin1") assert fs["price in \u00a5"] def test_from_response_multiple_forms_clickdata(self): @@ -737,7 +750,7 @@ class TestFormRequest(TestRequestBase): """ ) req = self.request_class.from_response(res) - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert fs == {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]} def test_from_response_radio(self): @@ -1022,7 +1035,7 @@ class TestFormRequest(TestRequestBase): with pytest.raises( ValueError, match="formdata should be a dict or iterable of tuples" ): - FormRequest.from_response(response, formdata=123) + FormRequest.from_response(response, formdata=123) # type: ignore[arg-type] def test_form_response_with_custom_invalid_formdata_value_error(self): """Test that a ValueError is raised for fault-inducing iterable formdata input""" @@ -1037,7 +1050,7 @@ class TestFormRequest(TestRequestBase): with pytest.raises( ValueError, match="formdata should be a dict or iterable of tuples" ): - FormRequest.from_response(response, formdata=("a",)) + FormRequest.from_response(response, formdata=("a",)) # type: ignore[arg-type] def test_get_form_with_xpath_no_form_parent(self): """Test that _get_from raised a ValueError when an XPath selects an element diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index 04315ad89..efa63e049 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -1,6 +1,7 @@ from __future__ import annotations import codecs +from typing import cast from unittest import mock import pytest @@ -14,6 +15,12 @@ from tests.utils.bases.http_response import TestResponseBase class TestTextResponse(TestResponseBase): response_class = TextResponse + def _links_response(self) -> TextResponse: + return cast("TextResponse", super()._links_response()) + + def _links_response_no_href(self) -> TextResponse: + return cast("TextResponse", super()._links_response_no_href()) + def test_follow_None_encoding(self): # unlike the base Response, TextResponse.follow() falls back to the # response encoding when encoding is None instead of raising @@ -21,7 +28,7 @@ class TestTextResponse(TestResponseBase): req = r.follow("foo", encoding=None) assert req.encoding == "cp1252" - def test_replace(self): + def test_replace(self) -> None: super().test_replace() r1 = self.response_class( "http://www.example.com", body="hello", encoding="cp852" @@ -344,7 +351,7 @@ class TestTextResponse(TestResponseBase): def test_follow_selector_list(self): resp = self._links_response() with pytest.raises(ValueError, match="SelectorList"): - resp.follow(resp.css("a")) + resp.follow(resp.css("a")) # type: ignore[arg-type] def test_follow_selector_invalid(self): resp = self._links_response() @@ -616,7 +623,7 @@ class CustomResponse(TextResponse): class TestCustomResponse(TestTextResponse): response_class = CustomResponse - def test_copy(self): + def test_copy(self) -> None: super().test_copy() r1 = self.response_class( url="https://example.org", @@ -632,7 +639,7 @@ class TestCustomResponse(TestTextResponse): assert r1.lost == "lost" assert r2.lost is None - def test_replace(self): + def test_replace(self) -> None: super().test_replace() r1 = self.response_class( url="https://example.org", diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index a624d2097..a2a4e8fdb 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -85,6 +85,7 @@ class TestCrawl: url = self.mockserver.url("/status?n=200") crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.request.url == url @@ -94,6 +95,7 @@ class TestCrawl: url = self.mockserver.url(f"/status?n={status}") crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) failure = crawler.spider.meta["failure"] response = failure.value.response assert failure.request.url == url @@ -111,6 +113,7 @@ class TestCrawl: }, ) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) failure = crawler.spider.meta["failure"] assert failure.request.url == url assert isinstance(failure.value, ZeroDivisionError) @@ -178,6 +181,7 @@ class TestCrawl: }, ) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.body == b"Caught ZeroDivisionError" assert response.request.url == OVERRIDDEN_URL @@ -201,6 +205,7 @@ class TestCrawl: }, ) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.body == b"Caught ZeroDivisionError" assert response.request.url == url diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index b88893b2b..6c26aa878 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -49,6 +49,7 @@ class InjectArgumentsSpiderMiddleware: async for element in result: if ( isinstance(element, Request) + and element.callback and element.callback.__name__ == "parse_spider_mw_2" ): element.cb_kwargs["from_process_spider_output"] = True @@ -68,7 +69,12 @@ class KeywordArgumentsSpider(MockServerSpider): checks: list[bool] = [] + def _inc_checks(self, count: int = 1) -> None: + assert self.crawler.stats + self.crawler.stats.inc_value("boolean_checks", count) + async def start(self): + assert self.mockserver data = {"key": "value", "number": 123, "callback": "some_callback"} yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data) yield Request( @@ -89,9 +95,10 @@ class KeywordArgumentsSpider(MockServerSpider): yield Request(self.mockserver.url("/spider_mw"), self.parse_spider_mw) def parse_first(self, response, key, number): + assert self.mockserver self.checks.append(key == "value") self.checks.append(number == 123) - self.crawler.stats.inc_value("boolean_checks", 2) + self._inc_checks(2) yield response.follow( self.mockserver.url("/two"), self.parse_second, @@ -100,28 +107,28 @@ class KeywordArgumentsSpider(MockServerSpider): def parse_second(self, response, new_key): self.checks.append(new_key == "new_value") - self.crawler.stats.inc_value("boolean_checks") + self._inc_checks() def parse_general(self, response, **kwargs): if response.url.endswith("/general_with"): self.checks.append(kwargs["key"] == "value") self.checks.append(kwargs["number"] == 123) self.checks.append(kwargs["callback"] == "some_callback") - self.crawler.stats.inc_value("boolean_checks", 3) + self._inc_checks(3) elif response.url.endswith("/general_without"): self.checks.append(kwargs == {}) - self.crawler.stats.inc_value("boolean_checks") + self._inc_checks() def parse_no_kwargs(self, response): self.checks.append(response.url.endswith("/no_kwargs")) - self.crawler.stats.inc_value("boolean_checks") + self._inc_checks() def parse_default(self, response, key, number=None, default=99): self.checks.append(response.url.endswith("/default")) self.checks.append(key == "value") self.checks.append(number == 123) self.checks.append(default == 99) - self.crawler.stats.inc_value("boolean_checks", 4) + self._inc_checks(4) def parse_takes_less(self, response, key, callback): """ @@ -140,17 +147,18 @@ class KeywordArgumentsSpider(MockServerSpider): ): self.checks.append(bool(from_process_request)) self.checks.append(bool(from_process_response)) - self.crawler.stats.inc_value("boolean_checks", 2) + self._inc_checks(2) def parse_spider_mw(self, response, from_process_spider_input, from_process_start): + assert self.mockserver self.checks.append(bool(from_process_spider_input)) self.checks.append(bool(from_process_start)) - self.crawler.stats.inc_value("boolean_checks", 2) + self._inc_checks(2) return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) def parse_spider_mw_2(self, response, from_process_spider_output): self.checks.append(bool(from_process_spider_output)) - self.crawler.stats.inc_value("boolean_checks", 1) + self._inc_checks() class TestCallbackKeywordArguments: diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 78ff18b15..c7596e45d 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,7 +1,10 @@ +from typing import Any + import pytest +from twisted.python.failure import Failure from scrapy import Request, Spider -from scrapy.http import JsonRequest +from scrapy.http import JsonRequest, Response from scrapy.utils.request import request_from_dict @@ -10,7 +13,7 @@ class CustomRequest(Request): class TestRequestSerialization: - def setup_method(self): + def setup_method(self) -> None: self.spider = MethodsSpider() def test_basic(self): @@ -42,12 +45,14 @@ class TestRequestSerialization: r = Request("http://www.example.com", body=b"\xc2\xa3") self._assert_serializes_ok(r) - def _assert_serializes_ok(self, request, spider=None): + def _assert_serializes_ok( + self, request: Request, spider: Spider | None = None + ) -> None: d = request.to_dict(spider=spider) request2 = request_from_dict(d, spider=spider) self._assert_same_request(request, request2) - def _assert_same_request(self, r1, r2): + def _assert_same_request(self, r1: Request, r2: Request) -> None: assert r1.__class__ == r2.__class__ assert r1.url == r2.url assert r1.callback == r2.callback @@ -64,6 +69,7 @@ class TestRequestSerialization: assert r1.dont_filter == r2.dont_filter assert r1.flags == r2.flags if isinstance(r1, JsonRequest): + assert isinstance(r2, JsonRequest) assert r1.dumps_kwargs == r2.dumps_kwargs def test_request_class(self): @@ -83,8 +89,8 @@ class TestRequestSerialization: def test_reference_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider.parse_item_reference, - errback=self.spider.handle_error_reference, + callback=self.spider.parse_item_reference, # type: ignore[arg-type,misc] + errback=self.spider.handle_error_reference, # type: ignore[arg-type,misc] ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) @@ -94,8 +100,8 @@ class TestRequestSerialization: def test_private_reference_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._MethodsSpider__parse_item_reference, - errback=self.spider._MethodsSpider__handle_error_reference, + callback=self.spider._MethodsSpider__parse_item_reference, # type: ignore[attr-defined] + errback=self.spider._MethodsSpider__handle_error_reference, # type: ignore[attr-defined] ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) @@ -105,7 +111,7 @@ class TestRequestSerialization: def test_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._MethodsSpider__parse_item_private, + callback=self.spider._MethodsSpider__parse_item_private, # type: ignore[attr-defined] errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -113,7 +119,7 @@ class TestRequestSerialization: def test_mixin_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._SpiderMixin__mixin_callback, + callback=self.spider._SpiderMixin__mixin_callback, # type: ignore[attr-defined] errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -127,7 +133,7 @@ class TestRequestSerialization: self._assert_serializes_ok(r, spider=self.spider) def test_unserializable_callback1(self): - r = Request("http://www.example.com", callback=lambda x: x) + r = Request("http://www.example.com", callback=lambda x: x) # type: ignore[misc] with pytest.raises( ValueError, match="is not an instance method in: None: pass spider = MySpider() r = Request("http://www.example.com", callback=spider.parse) - spider.parse = None + spider.parse = None # type: ignore[method-assign,assignment] with pytest.raises(ValueError, match="is not an instance method in: None: pass class SpiderDelegation: - def delegated_callback(self, response): + def delegated_callback(self, response: Response) -> None: pass -def parse_item(response): +def parse_item(response: Response) -> None: pass -def handle_error(failure): +def handle_error(failure: Failure) -> None: pass -def private_parse_item(response): +def private_parse_item(response: Response) -> None: pass -def private_handle_error(failure): +def private_handle_error(failure: Failure) -> None: pass @@ -197,15 +205,17 @@ class MethodsSpider(Spider, SpiderMixin): __parse_item_reference = private_parse_item __handle_error_reference = private_handle_error - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self.delegated_callback = SpiderDelegation().delegated_callback - def parse_item(self, response): + def parse_item(self, response: Response) -> None: pass - def handle_error(self, failure): + def handle_error(self, failure: Failure) -> None: pass - def __parse_item_private(self, response): # pylint: disable=unused-private-member + def __parse_item_private( # pylint: disable=unused-private-member + self, response: Response + ) -> None: pass diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 726e0573a..46a16ad1e 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -1,57 +1,62 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + from scrapy.signals import request_left_downloader from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.utils.decorators import inline_callbacks_test +if TYPE_CHECKING: + from scrapy import Request + from scrapy.crawler import Crawler + from tests.mockserver.http import MockServer + class SignalCatcherSpider(Spider): name = "signal_catcher" - def __init__(self, crawler, url, *args, **kwargs): + def __init__(self, crawler: Crawler, url: str, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) crawler.signals.connect(self.on_request_left, signal=request_left_downloader) self.caught_times = 0 self.start_urls = [url] @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler( + cls, crawler: Crawler, *args: Any, **kwargs: Any + ) -> SignalCatcherSpider: return cls(crawler, *args, **kwargs) - def on_request_left(self, request, spider): + def on_request_left(self, request: Request, spider: Spider) -> None: self.caught_times += 1 class TestCatching: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - @inline_callbacks_test - def test_success(self): + def test_success(self, mockserver: MockServer): crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl(self.mockserver.url("/status?n=200")) + yield crawler.crawl(mockserver.url("/status?n=200")) + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 @inline_callbacks_test - def test_timeout(self): + def test_timeout(self, mockserver: MockServer): crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1}) - yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) + yield crawler.crawl(mockserver.url("/delay?n=0.2")) + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 @inline_callbacks_test - def test_disconnect(self): + def test_disconnect(self, mockserver: MockServer): crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl(self.mockserver.url("/drop")) + yield crawler.crawl(mockserver.url("/drop")) + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 @inline_callbacks_test def test_noconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl("http://thereisdefinetelynosuchdomain.com") + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.py index 3a1e588ef..2b712f3ab 100644 --- a/tests/utils/bases/http_request.py +++ b/tests/utils/bases/http_request.py @@ -3,8 +3,9 @@ from abc import ABC, abstractmethod from typing import Any import pytest +from twisted.python.failure import Failure -from scrapy.http import Headers, Request +from scrapy.http import Headers, Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.utils.request import request_to_curl @@ -22,15 +23,15 @@ class TestRequestBase(ABC): def test_init(self): # Request requires url in the __init__ method with pytest.raises(TypeError): - self.request_class() + self.request_class() # type: ignore[call-arg] # url argument must be basestring with pytest.raises(TypeError): - self.request_class(123) + self.request_class(123) # type: ignore[arg-type] # priority argument must be an integer with pytest.raises(TypeError, match="Request priority not an integer"): - self.request_class("http://www.example.com", priority="1") + self.request_class("http://www.example.com", priority="1") # type: ignore[arg-type] r = self.request_class("http://www.example.com") assert isinstance(r.url, str) @@ -205,14 +206,17 @@ class TestRequestBase(ABC): def test_copy(self): """Test Request copy""" - def somecallback(): + def somecallback(response: Response) -> None: + pass + + def someerrback(failure: Failure) -> None: pass r1 = self.request_class( "http://www.example.com", flags=["f1", "f2"], callback=somecallback, - errback=somecallback, + errback=someerrback, ) r1.meta["foo"] = "bar" r1.cb_kwargs["key"] = "value" @@ -220,7 +224,7 @@ class TestRequestBase(ABC): # make sure callbaclks are copied assert r1.callback is somecallback - assert r1.errback is somecallback + assert r1.errback is someerrback assert r2.callback is r1.callback assert r2.errback is r1.errback @@ -251,7 +255,7 @@ class TestRequestBase(ABC): def test_copy_inherited_classes(self): """Test Request children copies preserve their class""" - class CustomRequest(self.request_class): + class CustomRequest(self.request_class): # type: ignore[misc,name-defined] pass r1 = CustomRequest("http://www.example.com") @@ -283,7 +287,9 @@ class TestRequestBase(ABC): assert r4.dont_filter is False # the cls argument allows changing the resulting class - custom_request_cls = type("CustomRequest", (self.request_class,), {}) + custom_request_cls: type[Request] = type( + "CustomRequest", (self.request_class,), {} + ) r5 = r1.replace(cls=custom_request_cls) assert isinstance(r5, custom_request_cls) assert r5.url == r1.url @@ -295,33 +301,36 @@ class TestRequestBase(ABC): def test_immutable_attributes(self): r = self.request_class("http://example.com") with pytest.raises(AttributeError): - r.url = "http://example2.com" + r.url = "http://example2.com" # type: ignore[misc] with pytest.raises(AttributeError): - r.body = "xxx" + r.body = "xxx" # type: ignore[misc,assignment] def test_callback_and_errback(self): - def a_function(): + def a_callback(response: Response) -> None: + pass + + def an_errback(failure: Failure) -> None: pass r1 = self.request_class("http://example.com") assert r1.callback is None assert r1.errback is None - r2 = self.request_class("http://example.com", callback=a_function) - assert r2.callback is a_function + r2 = self.request_class("http://example.com", callback=a_callback) + assert r2.callback is a_callback assert r2.errback is None - r3 = self.request_class("http://example.com", errback=a_function) + r3 = self.request_class("http://example.com", errback=an_errback) assert r3.callback is None - assert r3.errback is a_function + assert r3.errback is an_errback r4 = self.request_class( url="http://example.com", - callback=a_function, - errback=a_function, + callback=a_callback, + errback=an_errback, ) - assert r4.callback is a_function - assert r4.errback is a_function + assert r4.callback is a_callback + assert r4.errback is an_errback r5 = self.request_class( url="http://example.com", @@ -329,18 +338,18 @@ class TestRequestBase(ABC): errback=NO_CALLBACK, ) assert r5.callback is NO_CALLBACK - assert r5.errback is NO_CALLBACK + assert r5.errback is NO_CALLBACK # type: ignore[comparison-overlap] def test_callback_and_errback_type(self): with pytest.raises(TypeError): - self.request_class("http://example.com", callback="a_function") + self.request_class("http://example.com", callback="a_function") # type: ignore[arg-type] with pytest.raises(TypeError): - self.request_class("http://example.com", errback="a_function") + self.request_class("http://example.com", errback="a_function") # type: ignore[arg-type] with pytest.raises(TypeError): self.request_class( url="http://example.com", - callback="a_function", - errback="a_function", + callback="a_function", # type: ignore[arg-type] + errback="a_function", # type: ignore[arg-type] ) def test_setters(self): diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.py index 2fbf6527a..78e14e7b0 100644 --- a/tests/utils/bases/http_response.py +++ b/tests/utils/bases/http_response.py @@ -7,7 +7,7 @@ import pytest from w3lib.encoding import resolve_encoding from scrapy.exceptions import NotSupported -from scrapy.http import Headers, Request, Response +from scrapy.http import Headers, Request, Response, TextResponse from scrapy.link import Link from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS from tests import get_testdata @@ -15,6 +15,8 @@ from tests import get_testdata if TYPE_CHECKING: from collections.abc import Iterable + from parsel import Selector + class TestResponseBase(ABC): @property @@ -25,14 +27,14 @@ class TestResponseBase(ABC): def test_init(self): # Response requires url in the constructor with pytest.raises(TypeError): - self.response_class() + self.response_class() # type: ignore[call-arg] assert isinstance( self.response_class("http://example.com/"), self.response_class ) with pytest.raises(TypeError): - self.response_class(b"http://example.com") + self.response_class(b"http://example.com") # type: ignore[arg-type] with pytest.raises(TypeError): - self.response_class(url="http://example.com", body={}) + self.response_class(url="http://example.com", body={}) # type: ignore[arg-type] # body can be str or None assert isinstance( self.response_class("http://example.com/", body=b""), @@ -67,12 +69,12 @@ class TestResponseBase(ABC): r = self.response_class("http://www.example.com", status=301) assert r.status == 301 - r = self.response_class("http://www.example.com", status="301") + r = self.response_class("http://www.example.com", status="301") # type: ignore[arg-type] assert r.status == 301 with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): - self.response_class("http://example.com", status="lala200") + self.response_class("http://example.com", status="lala200") # type: ignore[arg-type] - def test_copy(self): + def test_copy(self) -> None: """Test Response copy""" r1 = self.response_class("http://www.example.com", body=b"Some body") @@ -121,7 +123,7 @@ class TestResponseBase(ABC): def test_copy_inherited_classes(self): """Test Response children copies preserve their class""" - class CustomResponse(self.response_class): + class CustomResponse(self.response_class): # type: ignore[misc,name-defined] pass r1 = CustomResponse("http://www.example.com") @@ -129,7 +131,7 @@ class TestResponseBase(ABC): assert isinstance(r2, CustomResponse) - def test_replace(self): + def test_replace(self) -> None: """Test Response.replace() method""" hdrs = Headers({"key": "value"}) r1 = self.response_class("http://www.example.com") @@ -146,7 +148,9 @@ class TestResponseBase(ABC): assert r4.body == b"" assert not r4.flags - def _assert_response_values(self, response, encoding, body): + def _assert_response_values( + self, response: TextResponse, encoding: str, body: str | bytes + ) -> None: if isinstance(body, str): body_unicode = body body_bytes = body.encode(encoding) @@ -160,15 +164,15 @@ class TestResponseBase(ABC): assert response.body == body_bytes assert response.text == body_unicode - def _assert_response_encoding(self, response, encoding): + def _assert_response_encoding(self, response: TextResponse, encoding: str) -> None: assert response.encoding == resolve_encoding(encoding) def test_immutable_attributes(self): r = self.response_class("http://example.com") with pytest.raises(AttributeError): - r.url = "http://example2.com" + r.url = "http://example2.com" # type: ignore[misc] with pytest.raises(AttributeError): - r.body = "xxx" + r.body = "xxx" # type: ignore[misc,assignment] def test_setter_mutable_lazy_loading(self): """Mutable attributes are set internally to None only until they are @@ -256,7 +260,7 @@ class TestResponseBase(ABC): def test_follow_None_url(self): r = self.response_class("http://example.com") with pytest.raises(ValueError, match="url can't be None"): - r.follow(None) + r.follow(None) # type: ignore[arg-type] def test_follow_None_encoding(self): r = self.response_class("http://example.com") @@ -325,20 +329,20 @@ class TestResponseBase(ABC): r = self.response_class("http://example.com") if self.response_class == Response: with pytest.raises(TypeError): - list(r.follow_all(urls=None)) + list(r.follow_all(urls=None)) # type: ignore[arg-type] with pytest.raises(TypeError): - list(r.follow_all(urls=12345)) + list(r.follow_all(urls=12345)) # type: ignore[arg-type] with pytest.raises(ValueError, match="url can't be None"): - list(r.follow_all(urls=[None])) + list(r.follow_all(urls=[None])) # type: ignore[list-item] else: with pytest.raises( ValueError, match="Please supply exactly one of the following arguments" ): - list(r.follow_all(urls=None)) + list(r.follow_all(urls=None)) # type: ignore[arg-type] with pytest.raises(TypeError): - list(r.follow_all(urls=12345)) + list(r.follow_all(urls=12345)) # type: ignore[arg-type] with pytest.raises(ValueError, match="url can't be None"): - list(r.follow_all(urls=[None])) + list(r.follow_all(urls=[None])) # type: ignore[list-item] @pytest.mark.xfail( not W3LIB_STRIPS_URLS, @@ -384,14 +388,14 @@ class TestResponseBase(ABC): def _assert_followed_url( self, - follow_obj: str | Link, + follow_obj: str | Link | Selector, target_url: str, response: Response | None = None, encoding: str | None = None, ) -> None: if response is None: response = self._links_response() - req = response.follow(follow_obj) + req = response.follow(follow_obj) # type: ignore[arg-type] assert req.url == target_url if encoding is not None: assert req.encoding == encoding From a7385d6e51034d60ea22eae218e2636a0c022a0d Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 3 Aug 2026 20:44:28 +0200 Subject: [PATCH 048/111] Type tests.mockserver and other resource-defining files (#7865) * Type tests.mockserver and other resource-defining files * Address pylint issues --- pyproject.toml | 4 - tests/mockserver/dns.py | 39 ++++- tests/mockserver/ftp.py | 20 ++- tests/mockserver/http.py | 85 ++++----- tests/mockserver/http_base.py | 10 +- tests/mockserver/http_resources.py | 174 +++++++++++-------- tests/mockserver/simple_https.py | 19 +- tests/test_core_downloader.py | 16 +- tests/test_downloader_handler_twisted_ftp.py | 22 ++- tests/test_http2_client_protocol.py | 22 +-- tox.ini | 3 +- 11 files changed, 260 insertions(+), 154 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 609c70708..13267e427 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,13 +118,10 @@ allow_incomplete_defs = true # 59 errors # TODO [[tool.mypy.overrides]] module = [ - "tests.mockserver.*", "tests.spiders", "tests.test_closespider", "tests.test_cmdline", "tests.test_contracts", - "tests.test_core_downloader", - "tests.test_downloader_handler_twisted_ftp", "tests.test_downloaderslotssettings", "tests.test_dupefilters", "tests.test_engine_loop", @@ -135,7 +132,6 @@ module = [ "tests.test_feedexport_postprocess", "tests.test_feedexport_storages", "tests.test_feedexport_uri_params", - "tests.test_http2_client_protocol", "tests.test_item", "tests.test_linkextractors", "tests.test_loader", diff --git a/tests/mockserver/dns.py b/tests/mockserver/dns.py index e19a2e61a..2af018c66 100644 --- a/tests/mockserver/dns.py +++ b/tests/mockserver/dns.py @@ -2,6 +2,7 @@ from __future__ import annotations import sys from subprocess import PIPE, Popen +from typing import TYPE_CHECKING from twisted.internet import defer from twisted.names import dns, error @@ -9,39 +10,63 @@ from twisted.names.server import DNSServerFactory from tests.utils import get_script_run_env +if TYPE_CHECKING: + from collections.abc import Sequence + from types import TracebackType + + from twisted.internet.defer import Deferred + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + +_Answers = tuple[list[dns.RRHeader], list[dns.RRHeader], list[dns.RRHeader]] + class MockDNSResolver: """ Implements twisted.internet.interfaces.IResolver partially """ - def _resolve(self, name): + def _resolve(self, name: bytes) -> _Answers: record = dns.Record_A(address=b"127.0.0.1") - answer = dns.RRHeader(name=name, payload=record) + # zope.interface has no type hints, so mypy cannot tell that Record_A + # provides the IEncodableRecord interface. + answer = dns.RRHeader(name=name, payload=record) # type: ignore[arg-type] return [answer], [], [] - def query(self, query, timeout=None): + def query( + self, query: dns.Query, timeout: Sequence[int] | None = None + ) -> Deferred[_Answers]: if query.type == dns.A: return defer.succeed(self._resolve(query.name.name)) return defer.fail(error.DomainError()) - def lookupAllRecords(self, name, timeout=None): + def lookupAllRecords( + self, name: bytes, timeout: Sequence[int] | None = None + ) -> Deferred[_Answers]: return defer.succeed(self._resolve(name)) class MockDNSServer: - def __enter__(self): + def __enter__(self) -> Self: self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver.dns"], stdout=PIPE, env=get_script_run_env(), text=True, ) + assert self.proc.stdout is not None self.host = "127.0.0.1" self.port = int(self.proc.stdout.readline().strip().split(":")[1]) return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: self.proc.kill() self.proc.communicate() @@ -54,7 +79,7 @@ def main() -> None: protocol = dns.DNSDatagramProtocol(controller=factory) listener = reactor.listenUDP(0, protocol) - def print_listening(): + def print_listening() -> None: host = listener.getHost() print(f"{host.host}:{host.port}") diff --git a/tests/mockserver/ftp.py b/tests/mockserver/ftp.py index 22efc966b..1edd64dda 100644 --- a/tests/mockserver/ftp.py +++ b/tests/mockserver/ftp.py @@ -7,6 +7,7 @@ from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp +from typing import TYPE_CHECKING from pyftpdlib.authorizers import DummyAuthorizer from pyftpdlib.handlers import FTPHandler @@ -14,6 +15,12 @@ from pyftpdlib.servers import FTPServer from tests.utils import get_script_run_env +if TYPE_CHECKING: + from types import TracebackType + + # typing.Self requires Python 3.11 + from typing_extensions import Self + class MockFTPServer: """Creates an FTP server on a random port with a default passwordless user @@ -26,7 +33,7 @@ class MockFTPServer: self.port: int | None = None self.path: Path | None = None - def __enter__(self): + def __enter__(self) -> Self: self.path = Path(mkdtemp()) self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)], @@ -34,6 +41,7 @@ class MockFTPServer: env=get_script_run_env(), text=True, ) + assert self.proc.stderr is not None for line in self.proc.stderr: if "starting FTP server" in line and ( m := re.search(r"starting FTP server on ([^ :]+):(\d+),", line) @@ -48,12 +56,18 @@ class MockFTPServer: ) return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: rmtree(str(self.path)) + assert self.proc is not None self.proc.kill() self.proc.communicate() - def url(self, path): + def url(self, path: str) -> str: return f"ftp://{self.host}:{self.port}/{path}" diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py index 7ad873c02..c4fd4464e 100644 --- a/tests/mockserver/http.py +++ b/tests/mockserver/http.py @@ -1,8 +1,8 @@ from __future__ import annotations from pathlib import Path +from typing import TYPE_CHECKING -from twisted.web import resource from twisted.web.static import Data, File from twisted.web.util import Redirect @@ -11,6 +11,7 @@ from tests import tests_datadir from .http_base import BaseMockServer, main_factory from .http_resources import ( ArbitraryLengthPayloadResource, + BaseResource, BrokenChunkedResource, BrokenDownloadResource, ChunkedResource, @@ -35,62 +36,68 @@ from .http_resources import ( SetCookie, Status, UriResource, + put_child, ) +if TYPE_CHECKING: + from twisted.web.server import Request -class Root(resource.Resource): - def __init__(self): + +class Root(BaseResource): + def __init__(self) -> None: super().__init__() - self.putChild(b"status", Status()) - self.putChild(b"follow", Follow()) - self.putChild(b"delay", Delay()) - self.putChild(b"partial", Partial()) - self.putChild(b"drop", Drop()) - self.putChild(b"raw", Raw()) - self.putChild(b"echo", Echo()) - self.putChild(b"payload", PayloadResource()) - self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) - self.putChild(b"static", File(str(Path(tests_datadir, "test_site/")))) - self.putChild(b"redirect-to", RedirectTo()) - self.putChild(b"text", Data(b"Works", "text/plain")) - self.putChild( + put_child(self, b"status", Status()) + put_child(self, b"follow", Follow()) + put_child(self, b"delay", Delay()) + put_child(self, b"partial", Partial()) + put_child(self, b"drop", Drop()) + put_child(self, b"raw", Raw()) + put_child(self, b"echo", Echo()) + put_child(self, b"payload", PayloadResource()) + put_child(self, b"alpayload", ArbitraryLengthPayloadResource()) + put_child(self, b"static", File(str(Path(tests_datadir, "test_site/")))) + put_child(self, b"redirect-to", RedirectTo()) + put_child(self, b"text", Data(b"Works", "text/plain")) + put_child( + self, b"html", Data( b"

Works

World

", "text/html", ), ) - self.putChild( + put_child( + self, b"enc-gb18030", Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), ) - self.putChild(b"redirect", Redirect(b"/redirected")) - self.putChild( - b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") + put_child(self, b"redirect", Redirect(b"/redirected")) + put_child( + self, b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") ) - self.putChild(b"redirected", Data(b"Redirected here", "text/plain")) + put_child(self, b"redirected", Data(b"Redirected here", "text/plain")) numbers = [str(x).encode("utf8") for x in range(2**18)] - self.putChild(b"numbers", Data(b"".join(numbers), "text/plain")) - self.putChild(b"wait", ForeverTakingResource()) - self.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) - self.putChild(b"host", HostHeaderResource()) - self.putChild(b"client-ip", ClientIPResource()) - self.putChild(b"broken", BrokenDownloadResource()) - self.putChild(b"chunked", ChunkedResource()) - self.putChild(b"broken-chunked", BrokenChunkedResource()) - self.putChild(b"contentlength", ContentLengthHeaderResource()) - self.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) - self.putChild(b"largechunkedfile", LargeChunkedFileResource()) - self.putChild(b"compress", Compress()) - self.putChild(b"duplicate-header", DuplicateHeaderResource()) - self.putChild(b"response-headers", ResponseHeadersResource()) - self.putChild(b"set-cookie", SetCookie()) - self.putChild(b"uri", UriResource()) + put_child(self, b"numbers", Data(b"".join(numbers), "text/plain")) + put_child(self, b"wait", ForeverTakingResource()) + put_child(self, b"hang-after-headers", ForeverTakingResource(write=True)) + put_child(self, b"host", HostHeaderResource()) + put_child(self, b"client-ip", ClientIPResource()) + put_child(self, b"broken", BrokenDownloadResource()) + put_child(self, b"chunked", ChunkedResource()) + put_child(self, b"broken-chunked", BrokenChunkedResource()) + put_child(self, b"contentlength", ContentLengthHeaderResource()) + put_child(self, b"nocontenttype", EmptyContentTypeHeaderResource()) + put_child(self, b"largechunkedfile", LargeChunkedFileResource()) + put_child(self, b"compress", Compress()) + put_child(self, b"duplicate-header", DuplicateHeaderResource()) + put_child(self, b"response-headers", ResponseHeadersResource()) + put_child(self, b"set-cookie", SetCookie()) + put_child(self, b"uri", UriResource()) - def getChild(self, path, request): + def getChild(self, path: bytes, request: Request) -> Root: return self - def render(self, request): + def render(self, request: Request) -> bytes: return b"Scrapy mock HTTP server\n" diff --git a/tests/mockserver/http_base.py b/tests/mockserver/http_base.py index 343c79781..5bc1252d6 100644 --- a/tests/mockserver/http_base.py +++ b/tests/mockserver/http_base.py @@ -17,6 +17,7 @@ from .utils import ssl_context_factory if TYPE_CHECKING: from collections.abc import Callable + from types import TracebackType from twisted.web import resource @@ -60,7 +61,12 @@ class BaseMockServer(ABC): self.https_port = https_parsed.port return self - def __exit__(self, exc_type, exc_value, traceback) -> None: + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: if self.proc: self.proc.kill() self.proc.communicate() @@ -135,7 +141,7 @@ def main_factory( context_factory = ssl_context_factory(**context_factory_kw) https_port = reactor.listenSSL(0, factory, context_factory) - def print_listening(): + def print_listening() -> None: if listen_http: http_host = http_port.getHost() http_address = f"http://{http_host.host}:{http_host.port}" diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py index 98ac6cf6a..cb028bc10 100644 --- a/tests/mockserver/http_resources.py +++ b/tests/mockserver/http_resources.py @@ -3,7 +3,7 @@ from __future__ import annotations import gzip import json import random -from typing import TYPE_CHECKING, ParamSpec, TypeVar +from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar from urllib.parse import urlencode from twisted.internet.task import deferLater @@ -14,17 +14,24 @@ from twisted.web.util import Redirect, redirectTo from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from twisted.internet.defer import Deferred - from twisted.web.http import Request + from twisted.python.failure import Failure + from twisted.web.http import Request as HTTPRequest + from twisted.web.server import Request _T = TypeVar("_T") _P = ParamSpec("_P") -def getarg(request, name, default=None, type_=None): +def getarg( + request: Request, + name: bytes, + default: Any = None, + type_: Callable[[bytes], Any] | None = None, +) -> Any: if name in request.args: value = request.args[name][0] if type_ is not None: @@ -33,73 +40,91 @@ def getarg(request, name, default=None, type_=None): return default -def close_connection(request): +def close_connection(request: Request) -> None: # We have to force a disconnection for HTTP/1.1 clients. Otherwise # client keeps the connection open waiting for more data. request.channel.loseConnection() request.finish() +def put_child(parent: resource.Resource, path: bytes, child: resource.Resource) -> None: + # zope.interface has no type hints, so mypy cannot tell that Resource + # instances provide the IResource interface that putChild() expects. + parent.putChild(path, child) # type: ignore[arg-type] + + +class BaseResource(resource.Resource): + """Base class for mockserver resources, with type hints.""" + + # Only needed to give subclasses a typed __init__ to call. + def __init__(self) -> None: # pylint: disable=useless-parent-delegation + super().__init__() # type: ignore[no-untyped-call] + + # most of the following resources are copied from twisted.web.test.test_webclient -class ForeverTakingResource(resource.Resource): +class ForeverTakingResource(BaseResource): """ L{ForeverTakingResource} is a resource which never finishes responding to requests. """ - def __init__(self, write=False): - resource.Resource.__init__(self) + def __init__(self, write: bool = False): + super().__init__() self._write = write - def render(self, request): + def render(self, request: Request) -> int: if self._write: request.write(b"some bytes") return server.NOT_DONE_YET -class HostHeaderResource(resource.Resource): +class HostHeaderResource(BaseResource): """ A testing resource which renders itself as the value of the host header from the request. """ - def render(self, request): - return request.requestHeaders.getRawHeaders(b"host")[0] + def render(self, request: Request) -> bytes: + headers = request.requestHeaders.getRawHeaders(b"host") + assert headers + return headers[0] -class ClientIPResource(resource.Resource): +class ClientIPResource(BaseResource): """ A testing resource which renders itself as the request client IP address. """ - def render(self, request): + def render(self, request: Request) -> bytes: client_address = request.getClientAddress() if client_address is None or client_address.host is None: return b"" return to_bytes(client_address.host) -class PayloadResource(resource.Resource): +class PayloadResource(BaseResource): """ A testing resource which renders itself as the contents of the request body as long as the request body is 100 bytes long, otherwise which renders itself as C{"ERROR"}. """ - def render(self, request): - data = request.content.read() - contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0] - if len(data) != 100 or int(contentLength) != 100: + def render(self, request: Request) -> bytes: + assert request.content + data: bytes = request.content.read() + content_length = request.requestHeaders.getRawHeaders(b"content-length") + assert content_length + if len(data) != 100 or int(content_length[0]) != 100: return b"ERROR" return data -class LeafResource(resource.Resource): +class LeafResource(BaseResource): isLeaf = True def deferRequest( self, - request: Request, + request: HTTPRequest, delay: float, f: Callable[_P, _T], *a: _P.args, @@ -107,7 +132,7 @@ class LeafResource(resource.Resource): ) -> Deferred[_T]: from twisted.internet import reactor - def _cancelrequest(_): + def _cancelrequest(_: Failure) -> None: # silence CancelledError d.addErrback(lambda _: None) d.cancel() @@ -118,12 +143,13 @@ class LeafResource(resource.Resource): class Follow(LeafResource): - def render(self, request): + def render(self, request: Request) -> int: total = getarg(request, b"total", 100, type_=int) show = getarg(request, b"show", 1, type_=int) order = getarg(request, b"order", b"desc") maxlatency = getarg(request, b"maxlatency", 0, type_=float) n = getarg(request, b"n", total, type_=int) + nlist: Sequence[int] if order == b"rand": nlist = [random.randint(1, total) for _ in range(show)] else: # order == "desc" @@ -133,7 +159,7 @@ class Follow(LeafResource): self.deferRequest(request, lag, self.renderRequest, request, nlist) return NOT_DONE_YET - def renderRequest(self, request, nlist): + def renderRequest(self, request: Request, nlist: Sequence[int]) -> None: s = """ """ args = request.args.copy() for nl in nlist: @@ -146,45 +172,47 @@ class Follow(LeafResource): class Delay(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> int: n = getarg(request, b"n", 1, type_=float) b = getarg(request, b"b", 1, type_=int) if b: # send headers now and delay body - request.write("") + request.write(b"") self.deferRequest(request, n, self._delayedRender, request, n) return NOT_DONE_YET - def _delayedRender(self, request, n): + def _delayedRender(self, request: Request, n: float) -> None: request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n")) request.finish() class Status(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> bytes: n = getarg(request, b"n", 200, type_=int) request.setResponseCode(n) return b"" class Raw(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> int: request.startedWriting = 1 self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET render_POST = render_GET - def _delayedRender(self, request): + def _delayedRender(self, request: Request) -> None: raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n") request.startedWriting = 1 request.write(raw) + assert request.channel.transport is not None request.channel.transport.loseConnection() request.finish() class Echo(LeafResource): - def render_GET(self, request): + def render_GET(self, request: Request) -> bytes: + assert request.content output = { "headers": { to_unicode(k): [to_unicode(v) for v in vs] @@ -198,27 +226,29 @@ class Echo(LeafResource): class RedirectTo(LeafResource): - def render(self, request): + def render(self, request: Request) -> bytes: goto = getarg(request, b"goto", b"/") # we force the body content, otherwise Twisted redirectTo() # returns HTML with int: request.setHeader(b"Content-Length", b"1024") self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET - def _delayedRender(self, request): + def _delayedRender(self, request: Request) -> None: request.write(b"partial content\n") request.finish() class Drop(Partial): - def _delayedRender(self, request): + def _delayedRender(self, request: Request) -> None: abort = getarg(request, b"abort", 0, type_=int) request.write(b"this connection will be dropped\n") tr = request.channel.transport @@ -233,8 +263,10 @@ class Drop(Partial): class ArbitraryLengthPayloadResource(LeafResource): - def render(self, request): - return request.content.read() + def render(self, request: Request) -> bytes: + assert request.content + data: bytes = request.content.read() + return data class NoMetaRefreshRedirect(Redirect): @@ -245,21 +277,23 @@ class NoMetaRefreshRedirect(Redirect): ) -class ContentLengthHeaderResource(resource.Resource): +class ContentLengthHeaderResource(BaseResource): """ A testing resource which renders itself as the value of the Content-Length header from the request. """ - def render(self, request): - return request.requestHeaders.getRawHeaders(b"content-length")[0] + def render(self, request: Request) -> bytes: + headers = request.requestHeaders.getRawHeaders(b"content-length") + assert headers + return headers[0] -class ChunkedResource(resource.Resource): - def render(self, request): +class ChunkedResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: request.write(b"chunked ") request.write(b"content\n") request.finish() @@ -268,11 +302,11 @@ class ChunkedResource(resource.Resource): return server.NOT_DONE_YET -class BrokenChunkedResource(resource.Resource): - def render(self, request): +class BrokenChunkedResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: request.write(b"chunked ") request.write(b"content\n") # Disable terminating chunk on finish. @@ -283,11 +317,11 @@ class BrokenChunkedResource(resource.Resource): return server.NOT_DONE_YET -class BrokenDownloadResource(resource.Resource): - def render(self, request): +class BrokenDownloadResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: request.setHeader(b"Content-Length", b"20") request.write(b"partial") close_connection(request) @@ -296,22 +330,24 @@ class BrokenDownloadResource(resource.Resource): return server.NOT_DONE_YET -class EmptyContentTypeHeaderResource(resource.Resource): +class EmptyContentTypeHeaderResource(BaseResource): """ A testing resource which renders itself as the value of request body without content-type header in response. """ - def render(self, request): + def render(self, request: Request) -> bytes: + assert request.content request.setHeader("content-type", "") - return request.content.read() + data: bytes = request.content.read() + return data -class LargeChunkedFileResource(resource.Resource): - def render(self, request): +class LargeChunkedFileResource(BaseResource): + def render(self, request: Request) -> int: from twisted.internet import reactor - def response(): + def response() -> None: for _ in range(1024): request.write(b"x" * 1024) request.finish() @@ -320,43 +356,45 @@ class LargeChunkedFileResource(resource.Resource): return server.NOT_DONE_YET -class DuplicateHeaderResource(resource.Resource): - def render(self, request): +class DuplicateHeaderResource(BaseResource): + def render(self, request: Request) -> bytes: request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"]) return b"" -class UriResource(resource.Resource): +class UriResource(BaseResource): """Return the full uri that was requested""" - def getChild(self, path, request): + def getChild(self, path: bytes, request: Request) -> resource.Resource: return self - def render(self, request): + def render(self, request: Request) -> bytes | int: # Note: this is an ugly hack for CONNECT request timeout test. # Returning some data here fail SSL/TLS handshake # ToDo: implement proper HTTPS proxy tests, not faking them. if request.method != b"CONNECT": return request.uri + assert request.transport is not None request.transport.write(b"HTTP/1.1 200 Connection established\r\n\r\n") return NOT_DONE_YET -class ResponseHeadersResource(resource.Resource): +class ResponseHeadersResource(BaseResource): """Return a response with headers set from the JSON request body""" - def render(self, request): + def render(self, request: Request) -> bytes: + assert request.content body = json.loads(request.content.read().decode()) for header_name, header_value in body.items(): request.responseHeaders.setRawHeaders(header_name, [header_value]) return json.dumps(body).encode("utf-8") -class Compress(resource.Resource): +class Compress(BaseResource): """Compress the data sent in the request url params and set Content-Encoding header""" - def render(self, request): - data = request.args.get(b"data")[0] + def render(self, request: Request) -> bytes: + data = request.args[b"data"][0] accept_encoding_header = request.getHeader(b"accept-encoding") @@ -370,10 +408,10 @@ class Compress(resource.Resource): return b"Did not receive a valid accept-encoding header" -class SetCookie(resource.Resource): +class SetCookie(BaseResource): """Return a response with a Set-Cookie header for each request url parameter""" - def render(self, request): + def render(self, request: Request) -> bytes: for cookie_name, cookie_values in request.args.items(): for cookie_value in cookie_values: cookie = (cookie_name.decode() + "=" + cookie_value.decode()).encode() diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.py index fdea666e1..2a6cb6dd8 100644 --- a/tests/mockserver/simple_https.py +++ b/tests/mockserver/simple_https.py @@ -2,18 +2,23 @@ from __future__ import annotations -from twisted.web import resource +from typing import TYPE_CHECKING + from twisted.web.static import Data from .http_base import BaseMockServer, main_factory +from .http_resources import BaseResource, put_child + +if TYPE_CHECKING: + from twisted.web.server import Request -class Root(resource.Resource): - def __init__(self): - resource.Resource.__init__(self) - self.putChild(b"file", Data(b"0123456789", "text/plain")) +class Root(BaseResource): + def __init__(self) -> None: + super().__init__() + put_child(self, b"file", Data(b"0123456789", "text/plain")) - def getChild(self, path, request): + def getChild(self, path: bytes, request: Request) -> Root: return self @@ -29,7 +34,7 @@ class SimpleMockServer(BaseMockServer): cipher_string: str | None = None, tls_min_version: str | None = None, tls_max_version: str | None = None, - ): + ) -> None: super().__init__() self.keyfile = keyfile self.certfile = certfile diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 3e4139b3e..912c0450b 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -14,7 +14,7 @@ from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse -from scrapy import Request +from scrapy import Request, Spider from scrapy.core.downloader import Downloader, Slot, tls from scrapy.core.downloader.contextfactory import ( _load_context_factory_from_settings, @@ -31,14 +31,17 @@ from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests.mockserver.http_resources import PayloadResource +from tests.mockserver.http_resources import PayloadResource, put_child from tests.mockserver.utils import ssl_context_factory from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from twisted.internet.defer import Deferred + from twisted.internet.interfaces import IListeningPort from twisted.web.iweb import IBodyProducer + from scrapy.http import Response + class TestSlot: def test_repr(self): @@ -52,7 +55,7 @@ class TestContextFactoryBase: async def server_url(self, tmp_path): (tmp_path / "file").write_bytes(b"0123456789") r = static.File(str(tmp_path)) - r.putChild(b"payload", PayloadResource()) + put_child(r, b"payload", PayloadResource()) site = server.Site(r, timeout=None) port = self._listen(site) portno = port.getHost().port @@ -61,7 +64,7 @@ class TestContextFactoryBase: await port.stopListening() - def _listen(self, site): + def _listen(self, site: server.Site) -> IListeningPort: from twisted.internet import reactor return reactor.listenSSL( @@ -317,7 +320,10 @@ def test_needs_backout(concurrency: int, active: int, expected: bool) -> None: @coroutine_test async def test_fetch_deprecated_spider_arg(): class CustomDownloader(Downloader): - def fetch(self, request, spider): # pylint: disable=signature-differs + # requiring the spider argument is what triggers the deprecation + def fetch( # type: ignore[override] # pylint: disable=signature-differs + self, request: Request, spider: Spider + ) -> Deferred[Response | Request]: return super().fetch(request, spider) crawler = get_crawler(DefaultSpider, {"DOWNLOADER": CustomDownloader}) diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py index 489b70e74..14de97b21 100644 --- a/tests/test_downloader_handler_twisted_ftp.py +++ b/tests/test_downloader_handler_twisted_ftp.py @@ -156,14 +156,17 @@ class TestFTP(TestFTPBase): for filename, content in self.test_files: (userdir / filename).write_bytes(content) - def _get_factory(self, root): + def _get_factory(self, root: Path) -> FTPFactory: from twisted.protocols.ftp import FTPFactory, FTPRealm realm = FTPRealm(anonymousRoot=str(root), userHome=str(root)) - p = portal.Portal(realm) + # zope.interface has no type hints, so mypy cannot tell that these + # objects provide the interfaces that Portal expects. + p = portal.Portal(realm) # type: ignore[arg-type] users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() - users_checker.addUser(self.username, self.password) - p.registerChecker(users_checker, credentials.IUsernamePassword) + # the FTP protocol authenticates with str credentials + users_checker.addUser(self.username, self.password) # type: ignore[arg-type] + p.registerChecker(users_checker, credentials.IUsernamePassword) # type: ignore[arg-type] return FTPFactory(portal=p) @deferred_f_from_coro_f @@ -192,12 +195,17 @@ class TestAnonymousFTP(TestFTPBase): for filename, content in self.test_files: (root / filename).write_bytes(content) - def _get_factory(self, tmp_path): + def _get_factory(self, tmp_path: Path) -> FTPFactory: from twisted.protocols.ftp import FTPFactory, FTPRealm realm = FTPRealm(anonymousRoot=str(tmp_path)) - p = portal.Portal(realm) - p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous) + # zope.interface has no type hints, so mypy cannot tell that these + # objects provide the interfaces that Portal expects. + p = portal.Portal(realm) # type: ignore[arg-type] + p.registerChecker( + checkers.AllowAnonymousAccess(), # type: ignore[arg-type] + credentials.IAnonymous, + ) return FTPFactory(portal=p, userAnonymous=self.username) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 3c1347fd3..b8586d1ca 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -30,7 +30,7 @@ from scrapy.utils.defer import ( deferred_from_coro, maybe_deferred_to_future, ) -from tests.mockserver.http_resources import LeafResource, Status +from tests.mockserver.http_resources import LeafResource, Status, put_child from tests.mockserver.utils import ssl_context_factory if TYPE_CHECKING: @@ -199,18 +199,18 @@ class TestHttps2ClientProtocol: @pytest.fixture def site(self, tmp_path): r = File(str(tmp_path)) - r.putChild(b"get-data-html-small", GetDataHtmlSmall()) - r.putChild(b"get-data-html-large", GetDataHtmlLarge()) + put_child(r, b"get-data-html-small", GetDataHtmlSmall()) + put_child(r, b"get-data-html-large", GetDataHtmlLarge()) - r.putChild(b"post-data-json-small", PostDataJsonSmall()) - r.putChild(b"post-data-json-large", PostDataJsonLarge()) + put_child(r, b"post-data-json-small", PostDataJsonSmall()) + put_child(r, b"post-data-json-large", PostDataJsonLarge()) - r.putChild(b"dataloss", Dataloss()) - r.putChild(b"no-content-length-header", NoContentLengthHeader()) - r.putChild(b"status", Status()) - r.putChild(b"query-params", QueryParams()) - r.putChild(b"timeout", TimeoutResponse()) - r.putChild(b"request-headers", RequestHeaders()) + put_child(r, b"dataloss", Dataloss()) + put_child(r, b"no-content-length-header", NoContentLengthHeader()) + put_child(r, b"status", Status()) + put_child(r, b"query-params", QueryParams()) + put_child(r, b"timeout", TimeoutResponse()) + put_child(r, b"request-headers", RequestHeaders()) return Site(r, timeout=None) @async_yield_fixture # type: ignore[untyped-decorator] diff --git a/tox.ini b/tox.ini index edde83356..ac32064a6 100644 --- a/tox.ini +++ b/tox.ini @@ -111,7 +111,8 @@ commands = pre-commit run {posargs:--all-files} [testenv:pylint] -basepython = python3 +# Some checks are Python-version-dependent, so pin the version used in CI. +basepython = python3.14 deps = {[testenv:extra-deps]deps} pylint==4.0.6 From 54da6c88aa5fb9812f7eb3baf7609c12b65a29f8 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 3 Aug 2026 20:46:41 +0200 Subject: [PATCH 049/111] Deprecate the download_delay spider attribute, and fix the suggested replacement for max_concurrent_requests (#7833) * Deprecate the download_delay and max_concurrent_requests spider attributes * Fix the deprecation entry of max_concurrent_requests --- docs/faq.rst | 16 ++--- docs/news.rst | 3 +- docs/topics/autothrottle.rst | 7 +- docs/topics/settings.rst | 4 -- extras/qpsclient.py | 19 +++--- scrapy/core/downloader/__init__.py | 31 ++------- scrapy/crawler.py | 28 ++++++++ scrapy/extensions/throttle.py | 16 ++--- tests/test_crawler.py | 33 +++++++++ tests/test_extension_throttle.py | 106 +++++++++++------------------ 10 files changed, 132 insertions(+), 131 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 1a574e5da..80658a5bf 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -220,21 +220,15 @@ the :ref:`topics-signals-ref` to know which ones. What does the response status code 999 mean? -------------------------------------------- -999 is a custom response status code used by Yahoo sites to throttle requests. +999 is a custom response status code used by some sites to throttle requests. Try slowing down the crawling speed by using a download delay of ``2`` (or -higher) in your spider: +higher) for the affected domains, with the :setting:`DOWNLOAD_SLOTS` setting: .. code-block:: python - from scrapy.spiders import CrawlSpider - - - class MySpider(CrawlSpider): - name = "myspider" - - download_delay = 2 - - # [ ... rest of the spider code ... ] + DOWNLOAD_SLOTS = { + "example.com": {"delay": 2}, + } Or by setting a global download delay in your project with the :setting:`DOWNLOAD_DELAY` setting. diff --git a/docs/news.rst b/docs/news.rst index 8f8477eaa..670843e0e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1417,7 +1417,8 @@ Deprecations - ``download_warnsize`` (use :setting:`DOWNLOAD_WARNSIZE`) - - ``max_concurrent_requests`` (use :setting:`CONCURRENT_REQUESTS`) + - ``max_concurrent_requests`` (use + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) - ``user_agent`` (use :setting:`USER_AGENT`) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 4f28019da..33289545c 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -106,10 +106,9 @@ delay of its download slot: Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) Note, however, that AutoThrottle still determines the starting delay of every -download slot by setting the ``download_delay`` attribute on the running -spider. If you want AutoThrottle not to impact a download slot at all, in -addition to setting this meta key in all requests that use that download slot, -you might want to set a custom value for the ``delay`` attribute of that +download slot. If you want AutoThrottle not to impact a download slot at all, +in addition to setting this meta key in all requests that use that download +slot, you might want to set a custom value for the ``delay`` attribute of that download slot, e.g. using :setting:`DOWNLOAD_SLOTS`. Settings diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e287c3bd5..65ee77258 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -953,10 +953,6 @@ desired. .. _spider-download_delay-attribute: -.. note:: - - This delay can be set per spider using :attr:`download_delay` spider attribute. - It is possible to change this setting per domain by using :setting:`DOWNLOAD_SLOTS`. diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 8e5001c1d..efb582254 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -16,23 +16,20 @@ class QPSSpider(Spider): name = "qps" benchurl = "http://localhost:8880/" - # Max concurrency is limited by global CONCURRENT_REQUESTS setting - max_concurrent_requests = 8 # Requests per second goal - qps = None # same as: 1 / download_delay - download_delay = None + qps = None # same as: 1 / DOWNLOAD_DELAY # time in seconds to delay server responses latency = None # number of slots to create slots = 1 - def __init__(self, *a, **kw): - super().__init__(*a, **kw) - if self.qps is not None: - self.qps = float(self.qps) - self.download_delay = 1 / self.qps - elif self.download_delay is not None: - self.download_delay = float(self.download_delay) + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + if spider.qps is not None: + spider.qps = float(spider.qps) + crawler.settings.set("DOWNLOAD_DELAY", 1 / spider.qps, priority="spider") + return spider async def start(self): url = self.benchurl diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index eb2079d0c..f9ee62838 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -27,7 +27,6 @@ from scrapy.utils.defer import ( deferred_from_coro, maybe_deferred_to_future, ) -from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: @@ -80,22 +79,6 @@ class Slot: ) -def _get_concurrency_delay( - concurrency: int, spider: Spider, settings: BaseSettings -) -> tuple[int, float]: - delay: float = settings.getfloat("DOWNLOAD_DELAY") - if hasattr(spider, "download_delay"): - delay = spider.download_delay - - if hasattr(spider, "max_concurrent_requests"): # pragma: no cover - warn_on_deprecated_spider_attribute( - "max_concurrent_requests", "CONCURRENT_REQUESTS" - ) - concurrency = spider.max_concurrent_requests - - return concurrency, delay - - class Downloader: DOWNLOAD_SLOT = "download_slot" _SLOT_GC_INTERVAL: float = 60.0 # seconds @@ -112,6 +95,9 @@ class Downloader: "CONCURRENT_REQUESTS_PER_DOMAIN" ) self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP") + # Default delay of new slots. AutoThrottle overrides it to apply + # AUTOTHROTTLE_START_DELAY. + self._delay: float = self.settings.getfloat("DOWNLOAD_DELAY") self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") self.middleware: DownloaderMiddlewareManager = ( DownloaderMiddlewareManager.from_crawler(crawler) @@ -147,16 +133,11 @@ class Downloader: ) -> tuple[str, Slot]: key = self.get_slot_key(request) if key not in self.slots: - assert self.crawler.spider slot_settings = self.per_slot_settings.get(key, {}) - conc = self.ip_concurrency or self.domain_concurrency - conc, delay = _get_concurrency_delay( - conc, self.crawler.spider, self.settings - ) - conc, delay = ( - slot_settings.get("concurrency", conc), - slot_settings.get("delay", delay), + conc = slot_settings.get( + "concurrency", self.ip_concurrency or self.domain_concurrency ) + delay = slot_settings.get("delay", self._delay) randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot diff --git a/scrapy/crawler.py b/scrapy/crawler.py index c8d74fba7..e2f726519 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -100,6 +100,10 @@ class Crawler: return self.addons.load_settings(self.settings) + self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY") + self._apply_deprecated_spider_attr( + "max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN" + ) self.stats = load_object(self.settings["STATS_CLASS"])(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) @@ -155,6 +159,30 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) + def _apply_deprecated_spider_attr(self, attr: str, setting: str) -> None: + """Bridge a deprecated spider attribute onto *setting*, warning about + the deprecation (and about being ignored when *setting* is already set + at spider or higher priority).""" + spider = self.spider if self.spider is not None else self.spidercls + if not hasattr(spider, attr): + return + if (self.settings.getpriority(setting) or 0) >= SETTINGS_PRIORITIES["spider"]: + warnings.warn( + f"The {attr!r} spider attribute is deprecated. It is also being " + f"ignored because {setting} is already set at spider or higher " + f"priority. Remove the {attr!r} attribute from your spider.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + return + warnings.warn( + f"The {attr!r} spider attribute is deprecated. Use the {setting} " + f"setting instead.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + self.settings.set(setting, getattr(spider, attr), priority="spider") + def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 542ff1cdc..cde73f12e 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -43,18 +43,18 @@ class AutoThrottle: return cls(crawler) def _spider_opened(self, spider: Spider) -> None: - self.mindelay = self._min_delay(spider) - self.maxdelay = self._max_delay(spider) - spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined] + self.mindelay = self._min_delay() + self.maxdelay = self._max_delay() + assert self.crawler.engine + self.crawler.engine.downloader._delay = self._start_delay() - def _min_delay(self, spider: Spider) -> float: - s = self.crawler.settings - return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY")) + def _min_delay(self) -> float: + return self.crawler.settings.getfloat("DOWNLOAD_DELAY") - def _max_delay(self, spider: Spider) -> float: + def _max_delay(self) -> float: return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") - def _start_delay(self, spider: Spider) -> float: + def _start_delay(self) -> float: return max( self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") ) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index b4f906e25..358f20ed7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -74,6 +74,39 @@ class TestCrawler: assert not settings.frozen assert crawler.settings.frozen + @pytest.mark.parametrize( + ("attr", "setting"), + [ + ("download_delay", "DOWNLOAD_DELAY"), + ("max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"), + ], + ) + def test_deprecated_spider_attr(self, attr: str, setting: str) -> None: + crawler = get_raw_crawler(type("_Spider", (DefaultSpider,), {attr: 2})) + with pytest.warns( + ScrapyDeprecationWarning, + match=f"The {attr!r} spider attribute is deprecated. Use the {setting} ", + ): + crawler._apply_settings() + assert crawler.settings.getint(setting) == 2 + + @pytest.mark.parametrize( + ("attr", "setting"), + [ + ("download_delay", "DOWNLOAD_DELAY"), + ("max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"), + ], + ) + def test_deprecated_spider_attr_ignored(self, attr: str, setting: str) -> None: + crawler = get_raw_crawler(type("_Spider", (DefaultSpider,), {attr: 2})) + crawler.settings.set(setting, 3, priority="spider") + with pytest.warns( + ScrapyDeprecationWarning, + match=f"The {attr!r} spider attribute is deprecated. It is also being ", + ): + crawler._apply_settings() + assert crawler.settings.getint(setting) == 3 + def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) assert crawler.settings["foo"] == "bar" diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index 4874f284a..2c718d95f 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -3,7 +3,7 @@ from unittest.mock import Mock import pytest -from scrapy import Request, Spider +from scrapy import Request from scrapy.exceptions import NotConfigured from scrapy.extensions.throttle import AutoThrottle from scrapy.http.response import Response @@ -25,6 +25,13 @@ def get_crawler(settings=None, spidercls=None): return _get_crawler(settings_dict=settings, spidercls=spidercls) +def _mock_downloader(crawler): + """Give *crawler* a mock engine, whose downloader AutoThrottle reads.""" + crawler.engine = Mock() + crawler.engine.downloader.slots = {} + return crawler.engine.downloader + + @pytest.mark.parametrize( ("value", "expected"), [ @@ -60,29 +67,21 @@ def test_target_concurrency_invalid(value): @pytest.mark.parametrize( - ("spider", "setting", "expected"), + ("setting", "expected"), [ - (UNSET, UNSET, DOWNLOAD_DELAY), - (1.0, UNSET, 1.0), - (UNSET, 1.0, 1.0), - (1.0, 2.0, 1.0), - (3.0, 2.0, 3.0), + (UNSET, DOWNLOAD_DELAY), + (1.0, 1.0), ], ) -def test_mindelay_definition(spider, setting, expected): +def test_mindelay_definition(setting, expected): settings = {} if setting is not UNSET: settings["DOWNLOAD_DELAY"] = setting - class _TestSpider(Spider): - name = "test" - - if spider is not UNSET: - _TestSpider.download_delay = spider - - crawler = get_crawler(settings, _TestSpider) + crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - at._spider_opened(_TestSpider()) + _mock_downloader(crawler) + at._spider_opened(DefaultSpider()) assert at.mindelay == expected @@ -99,58 +98,43 @@ def test_maxdelay_definition(value, expected): settings["AUTOTHROTTLE_MAX_DELAY"] = value crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + _mock_downloader(crawler) at._spider_opened(DefaultSpider()) assert at.maxdelay == expected @pytest.mark.parametrize( - ("min_spider", "min_setting", "start_setting", "expected"), + ("min_setting", "start_setting", "expected"), [ - (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), - (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), - (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), - (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), - (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), - (UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), - (UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0), - ( - AUTOTHROTTLE_START_DELAY + 1.0, - AUTOTHROTTLE_START_DELAY + 2.0, - UNSET, - AUTOTHROTTLE_START_DELAY + 1.0, - ), + (UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), + (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0), ( AUTOTHROTTLE_START_DELAY + 2.0, - UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 2.0, ), ( AUTOTHROTTLE_START_DELAY + 1.0, - UNSET, AUTOTHROTTLE_START_DELAY + 2.0, AUTOTHROTTLE_START_DELAY + 2.0, ), ], ) -def test_startdelay_definition(min_spider, min_setting, start_setting, expected): +def test_startdelay_definition(min_setting, start_setting, expected): settings = {} if min_setting is not UNSET: settings["DOWNLOAD_DELAY"] = min_setting if start_setting is not UNSET: settings["AUTOTHROTTLE_START_DELAY"] = start_setting - class _TestSpider(Spider): - name = "test" - - if min_spider is not UNSET: - _TestSpider.download_delay = min_spider - - crawler = get_crawler(settings, _TestSpider) + crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = _TestSpider() - at._spider_opened(spider) - assert spider.download_delay == expected + downloader = _mock_downloader(crawler) + at._spider_opened(DefaultSpider()) + assert downloader._delay == expected @pytest.mark.parametrize( @@ -174,15 +158,13 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) def test_skipped(meta, slot): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) request = Request("https://example.com", meta=meta) - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} if slot is not None: - crawler.engine.downloader.slots[slot] = object() + downloader.slots[slot] = object() at._adjust_delay = None # Raise exception if called. at._response_downloaded(None, request, spider) @@ -204,18 +186,16 @@ def test_adjustment(download_latency, target_concurrency, slot_delay, expected): settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url) - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = slot_delay - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot at._response_downloaded(response, request, spider) @@ -240,18 +220,16 @@ def test_adjustment_limits(mindelay, maxdelay, expected): } crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url) - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = slot_delay - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot at._response_downloaded(response, request, spider) @@ -272,18 +250,16 @@ def test_adjustment_bad_response( settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url, status=400) - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = slot_delay - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot at._response_downloaded(response, request, spider) @@ -294,19 +270,17 @@ def test_debug(caplog): settings = {"AUTOTHROTTLE_DEBUG": True} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": 1.0, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url, body=b"foo") - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = 2.0 slot.transferring = (None, None) - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot caplog.clear() with caplog.at_level(INFO): @@ -324,19 +298,17 @@ def test_debug(caplog): def test_debug_disabled(caplog): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) + downloader = _mock_downloader(crawler) spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": 1.0, "download_slot": "foo"} request = Request("https://example.com", meta=meta) response = Response(request.url, body=b"foo") - crawler.engine = Mock() - crawler.engine.downloader = Mock() - crawler.engine.downloader.slots = {} slot = Mock() slot.delay = 2.0 slot.transferring = (None, None) - crawler.engine.downloader.slots["foo"] = slot + downloader.slots["foo"] = slot caplog.clear() with caplog.at_level(INFO): From 388d4fcf04e3421f58cd7fc0920da28ee5449a00 Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 4 Aug 2026 08:56:46 +0200 Subject: [PATCH 050/111] Add a _load_objects() helper for object-or-path setting lists (#7857) Co-authored-by: Claude Opus 5 (1M context) --- scrapy/downloadermiddlewares/retry.py | 7 ++----- scrapy/utils/misc.py | 5 +++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index e1dcd90d5..f910d07c8 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import _warn_spider_arg -from scrapy.utils.misc import load_object +from scrapy.utils.misc import _load_objects from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message @@ -149,10 +149,7 @@ class RetryMiddleware: self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"] - self.exceptions_to_retry = tuple( - load_object(x) if isinstance(x, str) else x - for x in settings.getlist("RETRY_EXCEPTIONS") - ) + self.exceptions_to_retry = _load_objects(settings.getlist("RETRY_EXCEPTIONS")) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 20e7cb381..57b526be6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -90,6 +90,11 @@ def load_object(path: str | Callable[..., Any]) -> Any: return obj +def _load_objects(objects: Iterable[str | Callable[..., Any]]) -> tuple[Any, ...]: + """Resolve *objects* (objects or import paths) to a tuple of objects.""" + return tuple(load_object(obj) if isinstance(obj, str) else obj for obj in objects) + + def walk_modules_iter(path: str) -> Iterable[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that From 4cdde1c79c47ebc0480979d399154e70eabf2a93 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 4 Aug 2026 12:25:52 +0500 Subject: [PATCH 051/111] Bump CodSpeed to 5. (#7870) --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 82b16eea2..c50576ec7 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -53,7 +53,7 @@ jobs: uv tool install --with tox-uv tox tox -n -e benchmark - name: Run benchmarks - uses: CodSpeedHQ/action@f22792bfac16f3e14eb9fbea76f4a48e9cc22b93 # v4.19.1 + uses: CodSpeedHQ/action@0ca9cbbf4623b599a6c3ed4fc8a922942705d9f1 # v5.0.2 with: mode: simulation run: tox -e benchmark From 6e2081ca41960392cabdd5124fb4d4938cda311f Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 4 Aug 2026 11:41:25 +0200 Subject: [PATCH 052/111] Ask custom download handlers not to use engine.download_async() (#7871) --- docs/conf.py | 2 ++ docs/topics/download-handlers.rst | 24 +++------------------ scrapy/core/downloader/handlers/__init__.py | 16 ++++++++++++-- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index de722baac..1b41adaad 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -141,6 +141,8 @@ coverage_ignore_pyobjects = [ r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor", ] +# -- Options for the autodoc extension ---------------------------------------- +autodoc_member_order = "bysource" # -- Options for the InterSphinx extension ----------------------------------- # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 34ab4f105..e0501c169 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -78,33 +78,15 @@ Writing your own download handler A download handler is a :ref:`component ` that defines the following API: -.. class:: SampleDownloadHandler - - .. attribute:: lazy - :type: bool - - If ``False``, the handler will be instantiated when Scrapy is - initialized. - - If ``True``, the handler will only be instantiated when the first - request handled by it needs to be downloaded. - - .. method:: download_request(request: Request) -> Response - :async: - - Download the given request and return a response. - - .. method:: close() -> None - :async: - - Clean up any resources used by the handler. +.. autoclass:: scrapy.core.downloader.handlers.DownloadHandlerProtocol + :members: An optional base class for custom handlers is provided: .. autoclass:: scrapy.core.downloader.handlers.base.BaseDownloadHandler :members: :undoc-members: - :member-order: bysource + :exclude-members: close, download_request, lazy .. _download-handlers-exceptions: diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index fb27cdb8b..84dc6216b 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -39,11 +39,23 @@ logger = logging.getLogger(__name__) class DownloadHandlerProtocol(Protocol): + """Interface that :ref:`download handlers ` must + implement. + + Besides implementing this protocol, the contract of a download handler + includes **never** calling :meth:`crawler.engine.download_async() + `. + """ + lazy: bool + """Whether to delay instantiation of the handler; see :ref:`lazy + `.""" - async def download_request(self, request: Request) -> Response: ... + async def download_request(self, request: Request) -> Response: + """Download *request* and return a response.""" - async def close(self) -> None: ... + async def close(self) -> None: + """Clean up any resources used by the handler.""" class DownloadHandlers: From 0c542afa6589c0dd7a5a52a2469ae85fc1798928 Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 4 Aug 2026 15:47:00 +0200 Subject: [PATCH 053/111] Upgrade the minimum queuelib to 1.6.1 (#7874) --- pyproject.toml | 2 +- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 13267e427..0dcbade90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "parsel>=1.5.0", "protego>=0.1.15", "pyOpenSSL>=22.0.0", - "queuelib>=1.4.2", + "queuelib>=1.6.1", "service_identity>=23.1.0", "tldextract", "w3lib>=1.17.0", diff --git a/tox.ini b/tox.ini index ac32064a6..c56ad011b 100644 --- a/tox.ini +++ b/tox.ini @@ -143,7 +143,7 @@ deps = lxml==4.6.4 parsel==1.5.0 pyOpenSSL==22.0.0 - queuelib==1.4.2 + queuelib==1.6.1 service_identity==23.1.0 w3lib==1.17.0 zope.interface==5.1.0 @@ -261,7 +261,7 @@ deps = lxml==5.3.2 parsel==1.5.0 pyOpenSSL==24.3.0 - queuelib==1.4.2 + queuelib==1.6.1 service_identity==23.1.0 # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses # an inline flag placement that Python 3.11 treats as an error: global From 639fac78b310f3d7ce49cb6ed604bc3afc018e28 Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 4 Aug 2026 17:02:52 +0200 Subject: [PATCH 054/111] Cover the signature change of scrape_func (#7875) --- docs/news.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 670843e0e..a5cc6c723 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -2042,6 +2042,13 @@ Backward-incompatible changes ``process_start_requests()`` has been replaced by ``process_start()``. (:issue:`6729`) +- The ``scrape_func`` callable passed to + ``scrapy.core.spidermw.SpiderMiddlewareManager.scrape_response()`` is now + called with 2 parameters, ``response`` and ``request``, instead of 3, and + must return a :class:`~twisted.internet.defer.Deferred` instead of an + iterable. + (:issue:`6787`) + - The now-deprecated ``start_requests()`` method, when it returns an iterable instead of being defined as a generator, is now executed *after* the :ref:`scheduler ` instance has been created. From 91b70e4db48e622524827beb49e0ba33ef06b739 Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 4 Aug 2026 17:30:51 +0200 Subject: [PATCH 055/111] Improve the docs about scrapy parse --pipelines (#7876) --- docs/topics/commands.rst | 2 +- docs/topics/item-pipeline.rst | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 343193627..50da4593a 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -507,7 +507,7 @@ Supported options: * ``--cbkwargs``: additional keyword arguments that will be passed to the callback. This must be a valid json string. Example: --cbkwargs='{"foo" : "bar"}' -* ``--pipelines``: process items through pipelines +* ``--pipelines``: :ref:`process items through pipelines ` * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider` rules to discover the callback (i.e. spider method) to use for parsing the diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 951c0f485..c1313635c 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -330,6 +330,36 @@ passes through ``PricePipeline`` before it reaches the :ref:`feed exports .. _books.toscrape.com: https://books.toscrape.com/ +.. _test-item-pipeline: + +Testing an item pipeline +======================== + +To send the items from a single URL through your item pipelines, use the +:command:`parse` command with the ``--pipelines`` option:: + + scrapy parse --pipelines "https://books.toscrape.com/" + +To test specific item data instead, add a callback that builds an item out of +its keyword arguments: + +.. skip: next +.. code-block:: python + + class BooksSpider(scrapy.Spider): + # ... + + def parse_item(self, response, **fields): + yield BookItem(**fields) + +and pass those keyword arguments in the command line:: + + scrapy parse --pipelines -c parse_item --cbkwargs '{"title": "Test", "price": 10}' "https://books.toscrape.com/" + +Pass any URL that your spider handles; it is downloaded even though the +callback ignores it. + + Common pitfalls =============== From e0128c20c3e7328683e9f729bbc9aa105b39044f Mon Sep 17 00:00:00 2001 From: Adrian Date: Wed, 5 Aug 2026 08:51:22 +0200 Subject: [PATCH 056/111] Extend benchmarks (#7887) --- tests/benchmarks/__init__.py | 41 +++++++++++++- tests/benchmarks/test_crawl.py | 98 +++++++++++++++++++++++++++++++++- 2 files changed, 137 insertions(+), 2 deletions(-) diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py index 7b5ca0cb9..a066ed00a 100644 --- a/tests/benchmarks/__init__.py +++ b/tests/benchmarks/__init__.py @@ -1,14 +1,53 @@ from __future__ import annotations +import asyncio from typing import TYPE_CHECKING, Any +from scrapy.http import Response from scrapy.utils.test import get_crawler if TYPE_CHECKING: - from scrapy import Spider + from scrapy import Request, Spider from scrapy.crawler import Crawler +class NullDownloadHandler: + """Download handler that returns an empty response without doing any I/O. + + It lets benchmarks measure the engine, the scheduler and the middlewares + without also measuring HTTP parsing and socket handling, and reach as many + hostnames as they need without DNS resolution. + + It yields control to the event loop once per request, so that requests can + be in progress at the same time and concurrency limits apply. The peak + number of requests in progress is tracked in the + ``benchmark/peak_concurrency`` stat. + """ + + lazy = False + + def __init__(self, crawler: Crawler): + self._crawler = crawler + self._active = 0 + + @classmethod + def from_crawler(cls, crawler: Crawler) -> NullDownloadHandler: + return cls(crawler) + + async def download_request(self, request: Request) -> Response: + self._active += 1 + assert self._crawler.stats + self._crawler.stats.max_value("benchmark/peak_concurrency", self._active) + try: + await asyncio.sleep(0) + return Response(request.url, request=request) + finally: + self._active -= 1 + + async def close(self) -> None: + pass + + def crawl(spidercls: type[Spider], settings: dict[str, Any], **kwargs: Any) -> Crawler: """Run a crawl to completion and return its crawler. diff --git a/tests/benchmarks/test_crawl.py b/tests/benchmarks/test_crawl.py index 0fdfe742b..d3bf0fdd6 100644 --- a/tests/benchmarks/test_crawl.py +++ b/tests/benchmarks/test_crawl.py @@ -7,13 +7,14 @@ import pytest from scrapy import Field, Item, Request, Spider from scrapy.linkextractors import LinkExtractor -from tests.benchmarks import crawl +from tests.benchmarks import NullDownloadHandler, crawl if TYPE_CHECKING: from collections.abc import AsyncIterator from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found] + from scrapy.crawler import Crawler from scrapy.http import Response from tests.mockserver.http import MockServer @@ -22,6 +23,22 @@ pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspee PAGES = 100 LINKS_PER_PAGE = 5 +# Requests per crawl of the benchmarks that use NullDownloadHandler. The broad +# crawl scenarios split them differently between hostnames and pages per +# hostname. +REQUESTS = 200 +BROAD_DEEP_PAGES = 10 + +# Requests per crawl and delay of the benchmark that measures delayed requests, +# where wall time, unlike in the other benchmarks, is a function of the delay. +DELAYED_REQUESTS = 50 +DELAY = 0.005 + +NULL_SETTINGS: dict[str, Any] = { + "DOWNLOAD_HANDLERS": {"http": NullDownloadHandler}, + "LOG_ENABLED": False, +} + class _Page(Item): url = Field() @@ -45,11 +62,43 @@ class _FollowSpider(Spider): yield Request(link.url) +class _TreeSpider(Spider): + """Crawl *pages* pages on each of *domains* hostnames. + + Pages are numbered from 1, and page *n* links to pages *2n* and *2n+1*, so + that requests also reach the scheduler from callbacks, and not only from + :meth:`~scrapy.Spider.start`. + """ + + name = "benchmark-tree" + domains: int = 1 + pages: int = 1 + + async def start(self) -> AsyncIterator[Any]: + for domain in range(self.domains): + yield Request(f"http://d{domain}.example.com/1") + + def parse(self, response: Response) -> Any: + page = int(response.url.rpartition("/")[2]) + for child in (page * 2, page * 2 + 1): + if child <= self.pages: + yield Request(response.urljoin(f"/{child}")) + + class _Pipeline: def process_item(self, item: Any) -> Any: return item +def _crawl_tree(settings: dict[str, Any], *, domains: int, pages: int) -> Crawler: + crawler = crawl( + _TreeSpider, {**NULL_SETTINGS, **settings}, domains=domains, pages=pages + ) + assert crawler.stats + assert crawler.stats.get_value("downloader/response_count") == domains * pages + return crawler + + def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None: """Per-request overhead of a crawl over HTTP. @@ -67,3 +116,50 @@ def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> N assert crawler.stats.get_value("item_scraped_count") == PAGES + 1 benchmark(run) + + +def test_overhead_engine(benchmark: BenchmarkFixture) -> None: + """Per-request overhead of a crawl of a single hostname without any I/O.""" + + def run() -> None: + crawler = _crawl_tree({}, domains=1, pages=REQUESTS) + assert crawler.stats + assert crawler.stats.get_value("benchmark/peak_concurrency") > 1 + + benchmark(run) + + +@pytest.mark.parametrize( + ("domains", "pages"), + [ + pytest.param(REQUESTS, 1, id="shallow"), + pytest.param(REQUESTS // BROAD_DEEP_PAGES, BROAD_DEEP_PAGES, id="deep"), + ], +) +def test_overhead_broad(benchmark: BenchmarkFixture, domains: int, pages: int) -> None: + """Per-request overhead of a broad crawl. + + The shallow scenario, which reaches a single page of every hostname, pays + the cost of tracking a hostname for the first time on every request, and + gets its requests from :meth:`~scrapy.Spider.start`. The deep scenario, + which reaches the same number of pages spread over fewer hostnames, + amortizes that cost, and instead keeps several requests per hostname + waiting in the scheduler. + """ + benchmark(lambda: _crawl_tree({}, domains=domains, pages=pages)) + + +def test_overhead_concurrency(benchmark: BenchmarkFixture) -> None: + """Overhead of a crawl limited to 1 request at a time on a single hostname.""" + settings = {"CONCURRENT_REQUESTS_PER_DOMAIN": 1} + benchmark(lambda: _crawl_tree(settings, domains=1, pages=REQUESTS)) + + +def test_overhead_delay(benchmark: BenchmarkFixture) -> None: + """Overhead of a crawl where every request waits for a download delay. + + The delay is not randomized, so that wall time, and hence the number of + reactor iterations that the crawl needs, does not change between runs. + """ + settings = {"DOWNLOAD_DELAY": DELAY, "RANDOMIZE_DOWNLOAD_DELAY": False} + benchmark(lambda: _crawl_tree(settings, domains=1, pages=DELAYED_REQUESTS)) From 0c89e87b18adf77f4bcf8c21af2a43bc160ac828 Mon Sep 17 00:00:00 2001 From: Adrian Date: Wed, 5 Aug 2026 20:17:25 +0200 Subject: [PATCH 057/111] Switch tactics (#7895) --- .github/pull_request_template.md | 31 --- .github/workflows/auto-close-llm-pr.yml | 50 ----- .github/workflows/flag-prs-for-triage.yml | 220 ++++++++++++++++++++++ 3 files changed, 220 insertions(+), 81 deletions(-) delete mode 100644 .github/pull_request_template.md delete mode 100644 .github/workflows/auto-close-llm-pr.yml create mode 100644 .github/workflows/flag-prs-for-triage.yml diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 98a74f8ce..000000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,31 +0,0 @@ - diff --git a/.github/workflows/auto-close-llm-pr.yml b/.github/workflows/auto-close-llm-pr.yml deleted file mode 100644 index 15120b0d9..000000000 --- a/.github/workflows/auto-close-llm-pr.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Auto-close LLM PRs -# The workflow only reads the pull request body through the API, it never -# checks out or runs pull request code, so pull_request_target is safe here. -on: # zizmor: ignore[dangerous-triggers] - pull_request_target: - types: [opened] -permissions: - contents: read - pull-requests: write -jobs: - close-llm-pr: - name: Close PR if marked as LLM-written - runs-on: ubuntu-latest - steps: - - name: Check PR body and close if LLM-written - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const marker = "This PR was written entirely using an LLM"; - const { owner, repo } = context.repo; - const prNumber = context.payload.pull_request && context.payload.pull_request.number; - if (!prNumber) { - console.log('No pull request number found in context; exiting.'); - return; - } - const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber }); - const body = pr.body || ""; - if (body.includes(marker)) { - if (pr.state === 'closed') { - console.log(`PR #${prNumber} already closed.`); - return; - } - await github.rest.issues.addLabels({ - owner, - repo, - issue_number: prNumber, - labels: ['spam'] - }); - await github.rest.issues.createComment({ - owner, - repo, - issue_number: prNumber, - body: "Closing this PR because it contains the disclosure: \"This PR was written entirely using an LLM\"." - }); - await github.rest.pulls.update({ owner, repo, pull_number: prNumber, state: 'closed' }); - console.log(`Closed PR #${prNumber} because marker was found.`); - } else { - console.log(`Marker not found in PR #${prNumber}; nothing to do.`); - } diff --git a/.github/workflows/flag-prs-for-triage.yml b/.github/workflows/flag-prs-for-triage.yml new file mode 100644 index 000000000..662b88944 --- /dev/null +++ b/.github/workflows/flag-prs-for-triage.yml @@ -0,0 +1,220 @@ +name: Flag PRs for triage +# Labels pull requests whose author's public activity suggests that an LLM is +# writing them without supervision, and records the evidence in the workflow +# run summary so that triaging one does not require reading a user profile. +# +# Four independent signals, any of which is enough to label. Each one abstains +# when the data it needs is unavailable, so a missing signal never counts +# against an author: +# +# - Rejection burst: pull requests of theirs closed unmerged elsewhere within +# the last month. Volume of rejections in absolute terms separates spraying +# from ordinary contribution far better than a merge ratio does, since +# ratios reward authors who accumulate merges in trivial repositories. +# - Spray breadth: unrelated repositories they open pull requests against +# within one week. Breadth catches an agent on its first day, before any of +# its pull requests have been closed, and it comes from the event feed, so it +# also covers authors that the search API refuses to return. +# - Assistant voice: their recent comments across GitHub read as assistant +# output rather than as a developer talking, by section headings, bullet +# lists, em dash density or stock acknowledgement phrases. +# - Agent branch: the branch name carries an agent prefix. +# +# Deliberately not used: account age, fork age, follower count, total pull +# request count and cross-repository merge ratio. All of them were measured +# against hand-labelled pull requests and either failed to separate or, in the +# case of the merge ratio, inverted on held-out data. +# +# The label is advisory, and it says the author's history is worth a look +# before reviewing in depth; it does not say the pull request is bad. +# +# The workflow only reads pull request and public activity metadata through the +# API, it never checks out or runs pull request code, so pull_request_target is +# safe here. +on: # zizmor: ignore[dangerous-triggers] + pull_request_target: + types: [opened] +permissions: + contents: read + pull-requests: write +jobs: + flag-pr-for-triage: + name: Label PR if the author's activity suggests unsupervised LLM use + runs-on: ubuntu-latest + steps: + - name: Score the author and label the PR + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const LABEL = 'needs triage'; + const RETRIES = 5; + const RETRY_WAIT_MS = 60000; + const REJECTION_WINDOW_DAYS = 30; + const MIN_REJECTIONS = 1; + const MIN_COMMENTS = 2; + const MAX_REPOS_PER_WEEK = 2; + const VOICE = { structure: 0.10, emDashPerKChar: 0.30, acknowledgement: 0.40 }; + const EVENT_PAGES = 3; + const AGENT_BRANCH = /^(agent|codex|claude|cursor|devin|copilot|jules|bot)[\/_-]/i; + + const { owner, repo } = context.repo; + const pr = context.payload.pull_request; + const author = pr.user.login; + + if (pr.user.type === 'Bot' + || ['MEMBER', 'OWNER', 'COLLABORATOR'].includes(pr.author_association)) { + core.info(`Skipping PR #${pr.number} by ${author} (${pr.user.type}, ${pr.author_association}).`); + return; + } + + // Rate and abuse limits reset on the order of a minute, so waiting + // is enough; other errors are not worth retrying. + const retriable = new Set([403, 429, 500, 502, 503, 504]); + const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); + async function withRetries(description, call) { + for (let attempt = 1; ; attempt++) { + try { + return await call(); + } catch (error) { + if (!retriable.has(error.status) || attempt > RETRIES) throw error; + const reset = Number(error.response?.headers?.['x-ratelimit-reset']) * 1000 - Date.now(); + const after = Number(error.response?.headers?.['retry-after']) * 1000; + const wait = Math.min(Math.max(after || reset || RETRY_WAIT_MS, RETRY_WAIT_MS), 15 * RETRY_WAIT_MS); + core.info(`${description} failed with ${error.status}, retrying in ${Math.round(wait / 1000)}s (attempt ${attempt}/${RETRIES}).`); + await sleep(wait); + } + } + } + // Accounts excluded from search, deleted users and the like leave a + // signal unmeasurable rather than negative. + const orNull = promise => promise.catch(error => { + if ([404, 410, 422].includes(error.status)) return null; + throw error; + }); + + const opened = new Date(pr.created_at); + const daysBefore = date => (opened - new Date(date)) / 86400000; + + // Signal 1: pull requests closed unmerged elsewhere, recently. + const search = await orNull(withRetries('Searching for PRs by the author', () => + github.rest.search.issuesAndPullRequests({ + q: `author:${author} type:pr`, advanced_search: 'true', + sort: 'created', order: 'desc', per_page: 100, + }).then(response => response.data), + )); + let rejections = null; + if (search) { + rejections = search.items.filter(item => { + const itemOwner = item.repository_url.split('/repos/')[1].split('/')[0].toLowerCase(); + return itemOwner !== author.toLowerCase() + && item.state === 'closed' && !item.pull_request?.merged_at + && daysBefore(item.created_at) >= 0 + && daysBefore(item.created_at) <= REJECTION_WINDOW_DAYS; + }).map(item => item.html_url); + } + + // Signal 2: how their recent comments across GitHub read. + const events = []; + for (let page = 1; page <= EVENT_PAGES; page++) { + const batch = await orNull(withRetries(`Reading public events page ${page}`, () => + github.rest.activity.listPublicEventsForUser({ + username: author, per_page: 100, page, + }).then(response => response.data), + )); + if (!batch?.length) break; + events.push(...batch); + if (batch.length < 100) break; + } + const comments = events + .filter(event => ['IssueCommentEvent', 'PullRequestReviewCommentEvent'].includes(event.type)) + .map(event => event.payload?.comment?.body) + .filter(Boolean); + + // Signal 3: how many unrelated projects they open pull requests + // against in a single week. Breadth rather than volume: a focused + // contributor sends many pull requests to few repositories, while + // an unattended agent sprays a few across many. Taken from the + // event feed, which unlike search covers authors that search + // refuses to return. + const weeks = {}; + for (const event of events) { + if (event.type !== 'PullRequestEvent' || event.payload?.action !== 'opened') continue; + const name = event.repo?.name; + if (!name || name.toLowerCase().startsWith(`${author.toLowerCase()}/`)) continue; + const week = Math.floor(new Date(event.created_at) / (7 * 86400000)); + (weeks[week] ??= new Set()).add(name); + } + const breadth = events.length + ? Math.max(0, ...Object.values(weeks).map(repos => repos.size)) + : null; + const STRUCTURE = [/^\s*#{2,3}\s/m, /^\s*[-*]\s.+\n\s*[-*]\s/m, /\*\*[^*]+\*\*/, /```/]; + const ACKNOWLEDGEMENT = [ + /thanks for (the )?(review|feedback|pointing|catching|flagging|clarif)/i, + /you'?re (absolutely )?right/i, /great catch/i, /that makes sense/i, + /i'?ll (continue|investigate|update|submit|look into|make sure)/i, + /let me know (if|whether)/i, /happy to (update|adjust|revise|change)/i, + /i understand that/i, /thanks for your time/i, /just following up/i, + /hope (this|that) helps/i, /please let me know/i, /i'?ve (updated|addressed|fixed)/i, + ]; + let voice = null; + if (comments.length >= MIN_COMMENTS) { + const chars = comments.reduce((total, body) => total + body.length, 0); + const rate = patterns => comments.filter(body => patterns.some(re => re.test(body))).length / comments.length; + voice = { + comments: comments.length, + structure: rate(STRUCTURE), + acknowledgement: rate(ACKNOWLEDGEMENT), + emDashPerKChar: 1000 * comments.reduce((total, body) => total + (body.match(/—/g) || []).length, 0) / chars, + }; + } + + const reasons = []; + if (rejections && rejections.length >= MIN_REJECTIONS) { + reasons.push(`${rejections.length} PR(s) of theirs closed unmerged elsewhere in the last` + + ` ${REJECTION_WINDOW_DAYS} days: ${rejections.slice(0, 10).join(' ')}`); + } + if (voice && (voice.structure > VOICE.structure + || voice.emDashPerKChar > VOICE.emDashPerKChar + || voice.acknowledgement > VOICE.acknowledgement)) { + reasons.push(`comment style over ${voice.comments} recent comments:` + + ` ${(100 * voice.structure).toFixed(0)}% structured,` + + ` ${(100 * voice.acknowledgement).toFixed(0)}% stock acknowledgements,` + + ` ${voice.emDashPerKChar.toFixed(2)} em dashes per 1000 characters`); + } + if (breadth !== null && breadth > MAX_REPOS_PER_WEEK) { + reasons.push(`opened pull requests against ${breadth} unrelated repositories within a week`); + } + if (AGENT_BRANCH.test(pr.head?.ref || '')) { + reasons.push(`branch name carries an agent prefix: ${pr.head.ref}`); + } + + await core.summary + .addHeading(`PR #${pr.number} by ${author}`, 3) + .addList([ + rejections === null + ? 'recent rejections elsewhere: unmeasurable, the author cannot be searched' + : `recent rejections elsewhere: ${rejections.length}`, + voice === null + ? `comment style: unmeasurable, fewer than ${MIN_COMMENTS} recent comments found` + : `comment style: ${(100 * voice.structure).toFixed(0)}% structured,` + + ` ${(100 * voice.acknowledgement).toFixed(0)}% stock acknowledgements,` + + ` ${voice.emDashPerKChar.toFixed(2)} em dashes per 1000 characters` + + ` over ${voice.comments} comments`, + breadth === null + ? 'repositories per week: unmeasurable, no public events found' + : `repositories per week, at most: ${breadth}`, + `branch: ${pr.head?.ref ?? 'unknown'}`, + `verdict: ${reasons.length ? `labelled "${LABEL}"` : 'not labelled'}`, + ]) + .addRaw(reasons.length ? `\n${reasons.map(reason => `- ${reason}`).join('\n')}\n` : '') + .write(); + + if (!reasons.length) { + core.info(`Not labelling PR #${pr.number}.`); + return; + } + await withRetries('Adding the label', () => + github.rest.issues.addLabels({ owner, repo, issue_number: pr.number, labels: [LABEL] }), + ); + core.info(`Labelled PR #${pr.number}: ${reasons.join(' | ')}`); From 1bd839b57ddb614664a179b6213f49579bdfd3da Mon Sep 17 00:00:00 2001 From: Adrian Date: Thu, 6 Aug 2026 08:39:46 +0200 Subject: [PATCH 058/111] llm-check: allow-list authors by org or track record (#7906) --- .github/workflows/flag-prs-for-triage.yml | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/flag-prs-for-triage.yml b/.github/workflows/flag-prs-for-triage.yml index 662b88944..5ef8fa24a 100644 --- a/.github/workflows/flag-prs-for-triage.yml +++ b/.github/workflows/flag-prs-for-triage.yml @@ -20,6 +20,12 @@ name: Flag PRs for triage # lists, em dash density or stock acknowledgement phrases. # - Agent branch: the branch name carries an agent prefix. # +# Authors that the organisations behind this repository already trust are left +# alone before any of that runs: public members of those organisations, and +# authors with a track record of pull requests merged into their repositories. +# Trust from a merge record rather than from a list of names keeps the exemption +# in step with who is actually contributing. +# # Deliberately not used: account age, fork age, follower count, total pull # request count and cross-repository merge ratio. All of them were measured # against hand-labelled pull requests and either failed to separate or, in the @@ -56,6 +62,8 @@ jobs: const MAX_REPOS_PER_WEEK = 2; const VOICE = { structure: 0.10, emDashPerKChar: 0.30, acknowledgement: 0.40 }; const EVENT_PAGES = 3; + const TRUSTED_ORGS = ['scrapy', 'scrapy-plugins', 'scrapinghub', 'zytedata']; + const MIN_TRUSTED_MERGES = 10; const AGENT_BRANCH = /^(agent|codex|claude|cursor|devin|copilot|jules|bot)[\/_-]/i; const { owner, repo } = context.repo; @@ -93,6 +101,33 @@ jobs: throw error; }); + // author_association only reports membership of the organisation + // that owns this repository, and only when it is public, so trust + // in the author is established here instead. + const trustedOrg = (await Promise.all(TRUSTED_ORGS.map(org => + orNull(withRetries(`Checking public membership of ${org}`, () => + github.rest.orgs.checkPublicMembershipForUser({ org, username: author }), + )).then(response => response && org), + ))).find(Boolean); + if (trustedOrg) { + core.info(`Skipping PR #${pr.number} by ${author} (public member of ${trustedOrg}).`); + return; + } + // Repeating a qualifier narrows the search instead of widening it, + // hence the explicit disjunction. + const trustedMerges = await orNull(withRetries('Counting merged PRs in trusted organisations', () => + github.rest.search.issuesAndPullRequests({ + q: `author:${author} type:pr is:merged` + + ` (${TRUSTED_ORGS.map(org => `org:${org}`).join(' OR ')})`, + advanced_search: 'true', per_page: 1, + }).then(response => response.data.total_count), + )); + if (trustedMerges >= MIN_TRUSTED_MERGES) { + core.info(`Skipping PR #${pr.number} by ${author}` + + ` (${trustedMerges} PR(s) merged into ${TRUSTED_ORGS.join(', ')}).`); + return; + } + const opened = new Date(pr.created_at); const daysBefore = date => (opened - new Date(date)) / 86400000; From 9d2dea7a8df4f1c2cb59b71bca143148a5306c36 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 10:57:44 +0200 Subject: [PATCH 059/111] Stop skipping the IPv6 resolver tests (#7935) --- .../AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py | 5 ++++- tests/AsyncCrawlerProcess/default_name_resolver.py | 5 ++++- tests/CrawlerProcess/caching_hostname_resolver_ipv6.py | 5 ++++- tests/CrawlerProcess/default_name_resolver.py | 5 ++++- tests/test_crawler_subprocess.py | 7 +------ 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py index 55d2ef711..8181c4d17 100644 --- a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py @@ -8,7 +8,10 @@ class CachingHostnameResolverSpider(scrapy.Spider): """ name = "caching_hostname_resolver_spider" - start_urls = ["http://[::1]"] + + async def start(self): + # w3lib older than 2.4.1 strips the brackets, making the URL invalid. + yield scrapy.Request("http://[::1]", meta={"verbatim_url": True}) if __name__ == "__main__": diff --git a/tests/AsyncCrawlerProcess/default_name_resolver.py b/tests/AsyncCrawlerProcess/default_name_resolver.py index 4c8897f8f..7cc59594b 100644 --- a/tests/AsyncCrawlerProcess/default_name_resolver.py +++ b/tests/AsyncCrawlerProcess/default_name_resolver.py @@ -9,7 +9,10 @@ class IPv6Spider(scrapy.Spider): """ name = "ipv6_spider" - start_urls = ["http://[::1]"] + + async def start(self): + # w3lib older than 2.4.1 strips the brackets, making the URL invalid. + yield scrapy.Request("http://[::1]", meta={"verbatim_url": True}) if __name__ == "__main__": diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py index da9c16cb8..f6f865e3e 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py +++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py @@ -8,7 +8,10 @@ class CachingHostnameResolverSpider(scrapy.Spider): """ name = "caching_hostname_resolver_spider" - start_urls = ["http://[::1]"] + + async def start(self): + # w3lib older than 2.4.1 strips the brackets, making the URL invalid. + yield scrapy.Request("http://[::1]", meta={"verbatim_url": True}) if __name__ == "__main__": diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py index f4c129fdf..12b894030 100644 --- a/tests/CrawlerProcess/default_name_resolver.py +++ b/tests/CrawlerProcess/default_name_resolver.py @@ -9,7 +9,10 @@ class IPv6Spider(scrapy.Spider): """ name = "ipv6_spider" - start_urls = ["http://[::1]"] + + async def start(self): + # w3lib older than 2.4.1 strips the brackets, making the URL invalid. + yield scrapy.Request("http://[::1]", meta={"verbatim_url": True}) if __name__ == "__main__": diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index 733b6797d..240482586 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -10,9 +10,7 @@ from pathlib import Path from typing import TYPE_CHECKING import pytest -from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn -from w3lib import __version__ as w3lib_version from scrapy.utils.asyncio import sleep from tests.utils import get_script_run_env @@ -97,10 +95,6 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): ) assert "RuntimeError" not in log - @pytest.mark.skipif( - parse_version(w3lib_version) >= parse_version("2.0.0"), - reason="w3lib 2.0.0 and later do not allow invalid domains.", - ) def test_ipv6_default_name_resolver(self) -> None: log = self.run_script("default_name_resolver.py") assert "Spider closed (finished)" in log @@ -116,6 +110,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): def test_caching_hostname_resolver_ipv6(self) -> None: log = self.run_script("caching_hostname_resolver_ipv6.py") assert "Spider closed (finished)" in log + assert "http://::1" not in log assert "scrapy.exceptions.CannotResolveHostError" not in log def test_caching_hostname_resolver_finite_execution( From 63485522b619e9f338c1e11dbe5a81368ee618bb Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 10:58:59 +0200 Subject: [PATCH 060/111] Remove the job directory of a download slot once it drains (#7955) --- scrapy/pqueues.py | 7 +++++++ tests/test_pqueues.py | 25 ++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 41411ceaf..8e9783f5a 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -2,6 +2,8 @@ from __future__ import annotations import hashlib import logging +from contextlib import suppress +from pathlib import Path from typing import TYPE_CHECKING, Protocol, cast from scrapy.utils.misc import build_from_crawler @@ -409,6 +411,11 @@ class DownloaderAwarePriorityQueue: request = queue.pop() if len(queue) == 0: del self.pqueues[slot] + if self.key: + # Reclaim the slot directory; rmdir leaves it alone if the + # downstream queues did not remove all their files. + with suppress(OSError): + Path(self.key, _path_safe(slot)).rmdir() return request def push(self, request: Request) -> None: diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 85fefd172..6ecbb0721 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -6,7 +6,7 @@ import queuelib from scrapy.core.downloader import Downloader from scrapy.http.request import Request -from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue +from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue, _path_safe from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue from scrapy.utils.misc import build_from_crawler, load_object @@ -258,6 +258,29 @@ class TestDownloaderAwarePriorityQueue: assert "other-slot" not in self.queue +def test_slot_directory_removed_when_slot_drains(tmp_path): + crawler = get_crawler(Spider) + crawler.spider = crawler._create_spider("foo") + crawler.engine = Mock(downloader=MockDownloader()) + queue = DownloaderAwarePriorityQueue.from_crawler( + crawler=crawler, + downstream_queue_cls=PickleFifoDiskQueue, + key=str(tmp_path), + ) + request = Request("https://example.org/1") + slot_dir = tmp_path / _path_safe("example.org") + + queue.push(request) + assert slot_dir.is_dir() + + assert queue.pop().url == request.url + assert not slot_dir.exists() + + queue.push(request) + assert slot_dir.is_dir() + queue.close() + + @pytest.mark.parametrize( ("input_", "output"), [ From 1e92635a188315255ae57c68eb05fa142cfcd713 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:01:06 +0200 Subject: [PATCH 061/111] Benchmark item processing and item concurrency (#7954) --- tests/benchmarks/test_crawl.py | 115 +++++++++++++++++++++++++++++++-- 1 file changed, 110 insertions(+), 5 deletions(-) diff --git a/tests/benchmarks/test_crawl.py b/tests/benchmarks/test_crawl.py index d3bf0fdd6..f79b66a5b 100644 --- a/tests/benchmarks/test_crawl.py +++ b/tests/benchmarks/test_crawl.py @@ -1,5 +1,7 @@ from __future__ import annotations +import asyncio +from collections import Counter from typing import TYPE_CHECKING, Any from urllib.parse import urlencode @@ -29,11 +31,23 @@ LINKS_PER_PAGE = 5 REQUESTS = 200 BROAD_DEEP_PAGES = 10 -# Requests per crawl and delay of the benchmark that measures delayed requests, -# where wall time, unlike in the other benchmarks, is a function of the delay. +# Requests per crawl and delay of the benchmarks that wait, where wall time, +# unlike in the other benchmarks, is a function of the delay. DELAYED_REQUESTS = 50 DELAY = 0.005 +# Requests per crawl and items per response of the benchmarks that measure item +# processing, which reaches fewer pages than the other benchmarks because every +# page costs it several items. +ITEM_REQUESTS = 20 +ITEMS_PER_RESPONSE = 100 + +# Item concurrency limits of the benchmarks that measure item processing. The +# high limit is above the number of items that a response yields in any of +# them. +HIGH_CONCURRENT_ITEMS = 1000 +DELAYED_CONCURRENT_ITEMS = 50 + NULL_SETTINGS: dict[str, Any] = { "DOWNLOAD_HANDLERS": {"http": NullDownloadHandler}, "LOG_ENABLED": False, @@ -63,7 +77,8 @@ class _FollowSpider(Spider): class _TreeSpider(Spider): - """Crawl *pages* pages on each of *domains* hostnames. + """Crawl *pages* pages on each of *domains* hostnames, yielding *items* + items from every page. Pages are numbered from 1, and page *n* links to pages *2n* and *2n+1*, so that requests also reach the scheduler from callbacks, and not only from @@ -73,6 +88,7 @@ class _TreeSpider(Spider): name = "benchmark-tree" domains: int = 1 pages: int = 1 + items: int = 0 async def start(self) -> AsyncIterator[Any]: for domain in range(self.domains): @@ -83,6 +99,8 @@ class _TreeSpider(Spider): for child in (page * 2, page * 2 + 1): if child <= self.pages: yield Request(response.urljoin(f"/{child}")) + for _ in range(self.items): + yield _Page(url=response.url) class _Pipeline: @@ -90,12 +108,48 @@ class _Pipeline: return item -def _crawl_tree(settings: dict[str, Any], *, domains: int, pages: int) -> Crawler: +class _DelayedPipeline: + """Item pipeline that waits, so that the item concurrency limit applies. + + The peak number of items of a same response in progress is tracked in the + ``benchmark/peak_items`` stat. Items are counted per response because the + limit is per response, and the items of a response are processed while + later responses are already being downloaded. + """ + + def __init__(self, crawler: Crawler): + self._crawler = crawler + self._active: Counter[str] = Counter() + + @classmethod + def from_crawler(cls, crawler: Crawler) -> _DelayedPipeline: + return cls(crawler) + + async def process_item(self, item: Any) -> Any: + url = item["url"] + self._active[url] += 1 + assert self._crawler.stats + self._crawler.stats.max_value("benchmark/peak_items", self._active[url]) + try: + await asyncio.sleep(DELAY) + return item + finally: + self._active[url] -= 1 + + +def _crawl_tree( + settings: dict[str, Any], *, domains: int, pages: int, items: int = 0 +) -> Crawler: crawler = crawl( - _TreeSpider, {**NULL_SETTINGS, **settings}, domains=domains, pages=pages + _TreeSpider, + {**NULL_SETTINGS, **settings}, + domains=domains, + pages=pages, + items=items, ) assert crawler.stats assert crawler.stats.get_value("downloader/response_count") == domains * pages + assert crawler.stats.get_value("item_scraped_count", 0) == domains * pages * items return crawler @@ -163,3 +217,54 @@ def test_overhead_delay(benchmark: BenchmarkFixture) -> None: """ settings = {"DOWNLOAD_DELAY": DELAY, "RANDOMIZE_DOWNLOAD_DELAY": False} benchmark(lambda: _crawl_tree(settings, domains=1, pages=DELAYED_REQUESTS)) + + +@pytest.mark.parametrize( + ("items", "settings"), + [ + pytest.param(1, {}, id="single"), + pytest.param(ITEMS_PER_RESPONSE, {}, id="many"), + pytest.param( + 1, + {"CONCURRENT_ITEMS": HIGH_CONCURRENT_ITEMS}, + id="high-limit", + ), + ], +) +def test_overhead_items( + benchmark: BenchmarkFixture, items: int, settings: dict[str, Any] +) -> None: + """Overhead of sending the items of a callback through the item pipeline. + + The single and many scenarios, which use the default + :setting:`CONCURRENT_ITEMS` value, measure how that overhead grows with the + number of items that a response yields. The high-limit scenario instead + raises :setting:`CONCURRENT_ITEMS` well above that number. + """ + benchmark( + lambda: _crawl_tree(settings, domains=1, pages=ITEM_REQUESTS, items=items) + ) + + +def test_overhead_item_concurrency(benchmark: BenchmarkFixture) -> None: + """Overhead of a crawl where item processing waits. + + Every response yields more items than :setting:`CONCURRENT_ITEMS` allows in + parallel, so that the item pipeline gets them in several batches, and wall + time, unlike in most of the other benchmarks, is a function of the delay. + """ + settings = { + "CONCURRENT_ITEMS": DELAYED_CONCURRENT_ITEMS, + "ITEM_PIPELINES": {_DelayedPipeline: 100}, + } + + def run() -> None: + crawler = _crawl_tree( + settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE + ) + assert crawler.stats + assert ( + crawler.stats.get_value("benchmark/peak_items") == DELAYED_CONCURRENT_ITEMS + ) + + benchmark(run) From 56f4afd84e4a2b5d3c2957332fa79d373cdfbf52 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:03:57 +0200 Subject: [PATCH 062/111] Fix the telnet console shutdown error after a failed start (#7910) --- scrapy/extensions/telnet.py | 8 ++++++-- tests/test_extension_telnet.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 3be24c53f..1506cb1ea 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -52,6 +52,7 @@ class TelnetConsole(protocol.ServerFactory): self.crawler: Crawler = crawler self.noisy: bool = False + self.port: Port | None = None self.portrange: list[int] = [ int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT") ] @@ -71,7 +72,7 @@ class TelnetConsole(protocol.ServerFactory): return cls(crawler) def start_listening(self) -> None: - self.port: Port = listen_tcp(self.portrange, self.host, self) + self.port = listen_tcp(self.portrange, self.host, self) h = self.port.getHost() logger.info( "Telnet console listening on %(host)s:%(port)d", @@ -80,7 +81,10 @@ class TelnetConsole(protocol.ServerFactory): ) def stop_listening(self) -> None: - self.port.stopListening() + # The port is unset if start_listening() failed, e.g. because every + # port in TELNETCONSOLE_PORT was taken. + if self.port is not None: + self.port.stopListening() def protocol(self) -> telnet.TelnetTransport: class Portal: diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index fca0e3153..cf858e4ea 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -1,5 +1,6 @@ from __future__ import annotations +import socket from contextlib import contextmanager from typing import TYPE_CHECKING, Any @@ -91,6 +92,18 @@ def test_invalid_reversed_portrange() -> None: console.start_listening() +@coroutine_test +async def test_unavailable_port(caplog: pytest.LogCaptureFixture) -> None: + """Run a crawl where the console cannot bind any port.""" + with socket.create_server(("127.0.0.1", 0)) as sock: + port = sock.getsockname()[1] + crawler = _get_crawler(settings_dict={"TELNETCONSOLE_PORT": [port]}) + await crawler.crawl_async() + + assert "CannotListenError" in caplog.text + assert "AttributeError" not in caplog.text + + @coroutine_test async def test_telnet_vars() -> None: """Log into the console of a running crawl, which is when the telnet From 38f6e3cfd19ca844c0439b982f09a9ce04c30398 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:06:39 +0200 Subject: [PATCH 063/111] Remove the leftover KEEP_ALIVE setting from scrapy shell (#7928) --- scrapy/commands/shell.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 19138ffd0..52be1aadf 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -27,7 +27,6 @@ if TYPE_CHECKING: class Command(ScrapyCommand): default_settings: ClassVar[dict[str, Any]] = { "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", - "KEEP_ALIVE": True, "LOGSTATS_INTERVAL": 0, } From 5b4888a0b1e3dd5f235c3965f1a9144a868345f1 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:07:23 +0200 Subject: [PATCH 064/111] Document that signal handler order is undefined (#7941) --- docs/topics/item-pipeline.rst | 3 ++- docs/topics/signals.rst | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index c1313635c..35891ce8e 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -49,7 +49,8 @@ Additionally, they may also implement the following methods: .. method:: close_spider(self) - This method is called when the spider is closed. + This method is called when the spider is closed, before the + :signal:`spider_closed` signal is sent. Any of these methods may be defined as a coroutine function (``async def``). diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index f7f9f5cca..ceea2f7c0 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -44,6 +44,15 @@ Here is a simple example showing how you can catch signals and perform some acti def parse(self, response): pass +.. _signal-order: + +Handler order +============= + +The order in which the handlers of a signal run is undefined, and +:ref:`asynchronous handlers ` run concurrently. If two actions +must happen in a given order, run both from a single handler, in that order. + .. _signal-deferred: Asynchronous signal handlers From 81d12c6eb895e2026372dbb54a20e58597c69c5f Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:08:06 +0200 Subject: [PATCH 065/111] Document the Referer caveat of DEFAULT_REQUEST_HEADERS (#7917) --- docs/topics/settings.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 65ee77258..793d65f35 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -658,6 +658,11 @@ The default headers used for Scrapy HTTP Requests. They're populated in the :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. +.. caution:: A ``Referer`` header defined here only reaches requests for which + :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` does not set + one, such as start requests. To send it on every request, set + :setting:`REFERRER_POLICY` to ``"no-referrer"``. + .. setting:: DEPTH_LIMIT DEPTH_LIMIT From 4a69e48f0f648bd4509347f70963dd08bf3b28c8 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:08:49 +0200 Subject: [PATCH 066/111] Log the first depth-limited link only (#7916) --- docs/topics/stats.rst | 7 +++++++ scrapy/spidermiddlewares/depth.py | 14 +++++++++----- tests/test_spidermiddleware_depth.py | 17 +++++++++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index c702cefe7..b558c1cf2 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -121,6 +121,13 @@ one per actual value of the placeholder. :meth:`~scrapy.statscollectors.StatsCollector.get_stats` output is equivalent to a counter of 0. +.. stat:: depth/request_ignored_count + +``depth/request_ignored_count`` + Number of requests dropped for exceeding :setting:`DEPTH_LIMIT`. + + Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`. + .. stat:: downloader/exception_count ``downloader/exception_count`` diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 054804119..0131b62e7 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -41,6 +41,7 @@ class DepthMiddleware(BaseSpiderMiddleware): self.stats = stats self.verbose_stats = verbose_stats self.prio = prio + self._ignored_logged = False @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -94,11 +95,14 @@ class DepthMiddleware(BaseSpiderMiddleware): if self.prio: request.priority -= depth * self.prio if self.maxdepth and depth > self.maxdepth: - logger.debug( - "Ignoring link (depth > %(maxdepth)d): %(requrl)s ", - {"maxdepth": self.maxdepth, "requrl": request.url}, - extra={"spider": self.crawler.spider}, - ) + if not self._ignored_logged: + logger.debug( + f"Ignoring link (depth > {self.maxdepth}): {request.url}" + " - no more ignored links will be shown", + extra={"spider": self.crawler.spider}, + ) + self._ignored_logged = True + self.stats.inc_value("depth/request_ignored_count") return None if self.verbose_stats: self.stats.inc_value(f"request_depth_count/{depth}") diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index 32a2ea8f2..2aa76195e 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -85,6 +85,23 @@ async def test_process_spider_output_async_no_response( assert stats.get_value("request_depth_count/0") is None +def test_ignored_logged_once( + mw: DepthMiddleware, stats: StatsCollector, caplog: pytest.LogCaptureFixture +) -> None: + resp = Response("http://example.com") + resp.request = Request("http://example.com") + resp.meta["depth"] = 1 + result = [Request(f"http://example.com/{i}") for i in range(3)] + + with caplog.at_level("DEBUG", logger="scrapy.spidermiddlewares.depth"): + assert not list(mw.process_spider_output(resp, result)) + + messages = [r.getMessage() for r in caplog.records] + assert len(messages) == 1 + assert "http://example.com/0" in messages[0] + assert stats.get_value("depth/request_ignored_count") == 3 + + def test_priority_and_non_verbose_stats() -> None: crawler = get_crawler( Spider, From fc9c505e797591b45f82d0f7d918cd1eceffa5d1 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:12:44 +0200 Subject: [PATCH 067/111] Add URL benchmarks (#7914) --- tests/benchmarks/test_urls.py | 152 ++++++++++++++++++++++++++++++++++ tests/benchmarks/urls.txt | 130 +++++++++++++++++++++++++++++ 2 files changed, 282 insertions(+) create mode 100644 tests/benchmarks/test_urls.py create mode 100644 tests/benchmarks/urls.txt diff --git a/tests/benchmarks/test_urls.py b/tests/benchmarks/test_urls.py new file mode 100644 index 000000000..acc1d4d80 --- /dev/null +++ b/tests/benchmarks/test_urls.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from html import escape +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy import Request +from scrapy.http import HtmlResponse +from scrapy.linkextractors import LinkExtractor +from scrapy.utils.request import fingerprint + +if TYPE_CHECKING: + from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found] + +pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed") + +RESPONSE_URL = "https://www.example.com/catalogue/page-1.html" + +# Links that each scenario returns for the benchmark page. They are fewer than +# the anchors of the page because links to images, to other non-crawlable files +# and to non-HTTP schemes are rejected, and, except in the scenario that keeps +# duplicates, because the links that the navigation repeats are collapsed. +LINKS = 63 +DUPLICATE_LINKS = 88 +CANONICAL_LINKS = 60 +FILTERED_LINKS = 45 + +# Requests built from LINKS links that point to a different resource. +# Canonicalization maps the rest to one that another link already covers, e.g. +# two fragments of a page, or two spellings of one percent-escape. +FINGERPRINTS = 60 + + +def _read_corpus() -> tuple[list[str], list[str]]: + """Return the URLs of ``urls.txt``, and its first group of URLs. + + The first group is the site navigation, which the benchmark page repeats. + """ + groups: list[list[str]] = [[]] + for line in (Path(__file__).parent / "urls.txt").read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + if groups[-1]: + groups.append([]) + continue + groups[-1].append(line) + urls = [url for group in groups for url in group] + return urls, groups[0] + + +def _build_page(urls: list[str], navigation: list[str]) -> bytes: + """Return an HTML page that links to *urls*. + + Every link is surrounded by the markup of a product listing, so that + benchmarks also cover walking over the elements and attributes that a real + page puts between links. + """ + + def item(index: int, url: str) -> str: + href = escape(url) + return ( + f'
  • ' + f'Product {index}' + f'

    Product {index}

    ' + f'

    A description of product {index}.

    ' + f"
    • " + ) + + def nav(urls: list[str]) -> str: + links = "".join(f'{escape(url)}' for url in urls) + return f'' + + items = "".join(item(index, url) for index, url in enumerate(urls)) + return ( + "Catalogue" + f'' + f'{nav(navigation)}
      {items}
    {nav(navigation)}' + "" + ).encode() + + +URLS, NAVIGATION = _read_corpus() +BODY = _build_page(URLS, NAVIGATION) + + +def _response() -> HtmlResponse: + return HtmlResponse(RESPONSE_URL, body=BODY, encoding="utf-8") + + +@pytest.mark.parametrize( + ("kwargs", "links"), + [ + pytest.param({}, LINKS, id="default"), + pytest.param({"unique": False}, DUPLICATE_LINKS, id="duplicates"), + pytest.param({"canonicalize": True}, CANONICAL_LINKS, id="canonicalize"), + pytest.param( + { + "allow": r"/catalogue/", + "deny": r"/legal/", + "allow_domains": ["example.com", "www.example.com"], + }, + FILTERED_LINKS, + id="filtered", + ), + ], +) +def test_extract_links( + benchmark: BenchmarkFixture, kwargs: dict[str, Any], links: int +) -> None: + """Extraction of every link of a page. + + The scenarios cover the choices that change which work dominates: + deduplication and canonicalization both build a key for every link, and the + filters of a configured extractor reject links before the later checks, + which the default extractor reaches for every link. + """ + link_extractor = LinkExtractor(**kwargs) + + def run() -> None: + assert len(link_extractor.extract_links(_response())) == links + + benchmark(run) + + +EXTRACTED_URLS = [link.url for link in LinkExtractor().extract_links(_response())] + + +def test_requests(benchmark: BenchmarkFixture) -> None: + """Building a request for every link of a page.""" + + def run() -> None: + assert len([Request(url) for url in EXTRACTED_URLS]) == LINKS + + benchmark(run) + + +def test_fingerprints(benchmark: BenchmarkFixture) -> None: + """Fingerprinting the request of every link of a page. + + Requests are built here as well, and not once for all rounds, because + fingerprints are cached per request object. + """ + + def run() -> None: + assert ( + len({fingerprint(Request(url)) for url in EXTRACTED_URLS}) == FINGERPRINTS + ) + + benchmark(run) diff --git a/tests/benchmarks/urls.txt b/tests/benchmarks/urls.txt new file mode 100644 index 000000000..6ef6939f1 --- /dev/null +++ b/tests/benchmarks/urls.txt @@ -0,0 +1,130 @@ +# Link targets for the URL benchmarks, as they would appear in the href +# attribute of a page at https://www.example.com/catalogue/page-1.html. +# +# Cost per URL varies by shape: the number of query parameters drives the +# parsing and re-encoding of the query string, non-ASCII characters and +# unescaped characters drive percent-encoding, and non-default ports, dot +# segments and uppercase host names drive normalization. A corpus of uniform +# URLs would therefore measure one shape and miss the others, so this one +# covers each of them, in roughly the proportion of a real listing page. +# +# Blank lines and lines starting with "#" are ignored. + +# Site navigation. These also appear in a second copy of the navigation at the +# end of the page, so that deduplication has duplicates to collapse. +/ +/index.html +/about-us +/contact +/catalogue/ +/catalogue/page-2.html +/catalogue/page-3.html +/help/faq +/help/shipping-and-returns +/legal/terms +/legal/privacy + +# Relative paths of increasing depth. +detail.html +./detail.html +../catalogue/page-4.html +../../index.html +/catalogue/category/books/fiction/index.html +/catalogue/category/books/travel/mystery/historical/index.html +/a/b/c/d/e/f/g/h/i/j/k/index.html + +# One query parameter. +/catalogue/search?q=book +/catalogue/page-1.html?page=2 +/catalogue/detail?id=1042 + +# Several query parameters, in an order that canonicalization changes. +/catalogue/search?q=book&sort=price +/catalogue/search?sort=price&q=book +/catalogue/search?q=book&sort=price&page=3&per_page=20&in_stock=1 +/catalogue/search?zone=eu&q=book&min=10&max=90&sort=rating&page=2&view=grid&lang=en¤cy=EUR&ref=nav + +# Repeated keys, blank values and a bare key. +/catalogue/search?tag=fiction&tag=travel&tag=history +/catalogue/search?q=&sort= +/catalogue/search?featured + +# Characters that need percent-encoding. +/catalogue/search?q=cheap books +/catalogue/detail/a book about books.html +/catalogue/search?q=100%+cotton +/catalogue/search?price=%3E10&title=A%20%26%20B + +# Percent-escapes that are already valid, in both cases. +/catalogue/detail/%C3%A9dition-limit%C3%A9e.html +/catalogue/detail/%c3%a9dition-limit%c3%a9e.html +/catalogue/detail/%7Especial.html + +# Non-ASCII in the path and in the query. +/catalogue/detail/édition-limitée.html +/catalogue/search?q=édition +/catalogue/búsqueda?q=libro&categoría=ficción +/カタログ/詳細.html + +# Internationalized host names, encoded and decoded. +https://例え.テスト/catalogue/page-1.html +https://xn--r8jz45g.xn--zckzah/catalogue/page-2.html + +# Absolute URLs on the same host, on other hosts, and protocol-relative. +https://www.example.com/catalogue/page-5.html +https://www.example.com/catalogue/detail?id=1043 +http://www.example.com/catalogue/page-6.html +https://shop.example.com/catalogue/page-1.html +https://www.example.org/reviews/1042 +https://books.toscrape.com/catalogue/page-1.html +//cdn.example.com/catalogue/page-7.html +//www.example.com/catalogue/page-8.html + +# Ports, including the default one for the scheme. +https://www.example.com:443/catalogue/page-9.html +http://www.example.com:80/catalogue/page-10.html +https://staging.example.com:8443/catalogue/page-1.html + +# Host name case, which normalization lowercases. +https://WWW.EXAMPLE.COM/catalogue/Page-11.html +HTTPS://www.example.com/catalogue/page-12.html + +# Dot segments, empty segments and trailing slashes, which WHATWG +# normalization resolves and the standard library keeps. +/catalogue/../catalogue/page-13.html +/catalogue/./page-14.html +/catalogue//page-15.html +/catalogue/category/ +/catalogue/category + +# Fragments, which canonicalization drops and the deduplication key keeps. +/catalogue/page-16.html#reviews +/catalogue/page-16.html#description +/catalogue/page-17.html# +#top + +# Path parameters, where the semicolon is not the last segment. +/catalogue;sessionid=abc123/page-18.html +/catalogue/page-19.html;sessionid=abc123 + +# User information in the authority. +https://user:password@files.example.com/catalogue/page-1.html + +# A long URL, of the length that tracking parameters reach. +/catalogue/search?q=book&utm_source=newsletter&utm_medium=email&utm_campaign=spring-sale-2026&utm_term=fiction%20paperback&utm_content=hero-banner-variant-b&session=6f1c9a2e4b7d8f0a1c3e5d7b9f2a4c6e&ref=https%3A%2F%2Fwww.example.org%2Freviews%2F1042&page=2&sort=relevance + +# Extensions that the default deny_extensions rejects, and one compound +# extension, which only matches as a whole. +/media/cover-1042.jpg +/media/cover-1042.PNG +/media/catalogue.pdf +/static/style.css +/static/app.js +/downloads/catalogue.tar.gz +/downloads/catalogue.zip + +# Schemes that are not crawlable, which are rejected before any parsing. +mailto:orders@example.com +javascript:void(0) +tel:+441234567890 +data:text/plain,hello From b4279e243bbdf00054bb9c724d9fb4db2766068e Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:13:29 +0200 Subject: [PATCH 068/111] Document that process_value runs before allow and deny (#7940) --- scrapy/linkextractors/lxmlhtml.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 3fb741d7a..46ac39a28 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -282,6 +282,12 @@ class LxmlLinkExtractor: if m: return m.group(1) + ``process_value`` is called before the filtering parameters, such as + ``allow`` and ``deny``, which match the value that it returns. To drop + links based on their final URL, use the ``process_links`` parameter of + :class:`~scrapy.spiders.Rule`, which only receives links that those + parameters kept. + :type process_value: collections.abc.Callable :param strip: whether to strip whitespaces from extracted attributes. From 94dff69468a9e734e10e0feea2674ae1ebf46bf7 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:15:58 +0200 Subject: [PATCH 069/111] Add VCS CI job, fix support for upcoming parsel version (#7924) --- .github/workflows/tests-vcs-deps.yml | 53 ++++++++++++++++++++++++++++ scrapy/selector/unified.py | 13 +++++-- tox.ini | 45 +++++++++++++++++++++++ 3 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/tests-vcs-deps.yml diff --git a/.github/workflows/tests-vcs-deps.yml b/.github/workflows/tests-vcs-deps.yml new file mode 100644 index 000000000..bf867dba7 --- /dev/null +++ b/.github/workflows/tests-vcs-deps.yml @@ -0,0 +1,53 @@ +name: VCS dependencies + +permissions: + contents: read + +on: + schedule: + - cron: '0 4 * * *' + workflow_dispatch: + +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + +jobs: + tests: + name: tests + runs-on: ubuntu-latest + env: + PYTEST_ADDOPTS: -n auto --no-cov + TOXENV: vcs-deps + UV_PYTHON_PREFERENCE: only-system + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + # Dependencies that ship wheels on PyPI are built from source here, so + # their build dependencies are needed: libxml2 and libxslt for lxml, + # libjpeg and zlib for Pillow, and autotools for the libuv bundled in + # uvloop. + - name: Install system libraries + run: | + sudo apt-get update + sudo apt-get install automake libjpeg-dev libtool libxml2-dev libxslt-dev zlib1g-dev + + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install mitmproxy + run: uv tool install --python cpython mitmproxy + + - name: Run tests + run: uvx --with tox-uv tox diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index f6334c32c..fa91e2904 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -46,8 +46,10 @@ class Selector(_ParselSelector, object_ref): ``"json"``, ``"text"`` or ``None`` (default). It's passed to :class:`parsel.Selector` and its meaning is defined there. However, when ``type`` is ``None``, it is set to ``"xml"`` for an - :class:`~scrapy.http.XmlResponse` and to ``"html"`` otherwise before - passing it to :class:`parsel.Selector`. + :class:`~scrapy.http.XmlResponse` and to ``"html"`` for an + :class:`~scrapy.http.HtmlResponse` or for ``text`` before passing it to + :class:`parsel.Selector`, which for any other response is left to + determine the type from the response body. .. note:: JSON selector support requires ``parsel`` 1.8.0 or higher. With older versions setting ``type`` to ``"json"`` or ``"text"`` is not @@ -70,8 +72,13 @@ class Selector(_ParselSelector, object_ref): f"{self.__class__.__name__}.__init__() received both response and text" ) + # A response that is neither HTML nor XML, e.g. a JSON one, keeps type + # unset, so that parsel determines it from the body. if type is None: - type = "xml" if isinstance(response, XmlResponse) else "html" # noqa: A001 + if isinstance(response, XmlResponse): + type = "xml" # noqa: A001 + elif response is None or isinstance(response, HtmlResponse): + type = "html" # noqa: A001 if text is not None: response = _response_from_text(text, type) diff --git a/tox.ini b/tox.ini index c56ad011b..94c04f75f 100644 --- a/tox.ini +++ b/tox.ini @@ -201,6 +201,51 @@ setenv = {[min]setenv} commands = {[min]commands} +[testenv:vcs-deps] +basepython = python3 +deps = + {[testenv:extra-deps]deps} + uv +# Dependencies cap each other at their latest release, so their development +# branches usually cannot be resolved together: pyOpenSSL, for one, requires a +# cryptography older than the one cryptography itself is heading towards. +# --no-deps skips resolution entirely, replacing only these distributions and +# leaving the rest of the environment as the install above resolved it. +# +# Pillow and uvloop build from source, and need the libjpeg headers and +# autotools respectively. robotexclusionrulesparser has no public repository, +# so it stays at its latest release. +commands_pre = + uv pip install --python {envpython} --no-deps --reinstall \ + git+https://github.com/twisted/twisted \ + git+https://github.com/python-pillow/Pillow \ + git+https://github.com/MagicStack/uvloop \ + git+https://github.com/pyca/cryptography \ + git+https://github.com/scrapy/cssselect \ + git+https://github.com/tiran/defusedxml \ + git+https://github.com/scrapy/itemadapter \ + git+https://github.com/scrapy/itemloaders \ + git+https://github.com/lxml/lxml \ + git+https://github.com/pypa/packaging \ + git+https://github.com/scrapy/parsel \ + git+https://github.com/scrapy/protego \ + git+https://github.com/pyca/pyopenssl \ + git+https://github.com/scrapy/queuelib \ + git+https://github.com/pyca/service-identity \ + git+https://github.com/john-kurkowski/tldextract \ + git+https://github.com/scrapy/w3lib \ + git+https://github.com/zopefoundation/zope.interface \ + git+https://github.com/mcfletch/pydispatcher \ + git+https://github.com/boto/boto3 \ + git+https://github.com/bpython/bpython \ + git+https://github.com/google/brotli \ + git+https://github.com/python-hyper/brotlicffi \ + git+https://github.com/googleapis/python-storage \ + git+https://github.com/pydantic/httpx2\#subdirectory=src/httpx2 \ + git+https://github.com/ipython/ipython \ + git+https://github.com/prompt-toolkit/ptpython \ + git+https://github.com/indygreg/python-zstandard + [testenv:default-reactor] commands = {[testenv]commands} --reactor=default From 0e324f3d4a5576c51423c506ef554db5717ce603 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:17:44 +0200 Subject: [PATCH 070/111] Let spiders change allowed_domains at run time (#7912) --- docs/topics/spiders.rst | 7 ++++ scrapy/downloadermiddlewares/offsite.py | 11 ++++- tests/test_downloadermiddleware_offsite.py | 47 ++++++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 8fbf0c52d..f2cfeb712 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -59,9 +59,16 @@ scrapy.Spider :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is enabled. + .. versionchanged:: VERSION + Changes to this attribute during a crawl are now taken into account. + Let's say your target url is ``https://www.example.com/1.html``, then add ``'example.com'`` to the list. + You may modify this attribute while the spider runs, e.g. to allow + domains that you only learn about from an earlier response. The change + affects requests scheduled after it. + .. autoattribute:: start_urls .. attribute:: custom_settings diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index db85b62a1..28c0e09cb 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -22,10 +22,12 @@ logger = logging.getLogger(__name__) class OffsiteMiddleware: crawler: Crawler + host_regex: re.Pattern[str] def __init__(self, stats: StatsCollector): self.stats = stats self.domains_seen: set[str] = set() + self._allowed_domains: list[str] | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -37,7 +39,13 @@ class OffsiteMiddleware: return o def spider_opened(self, spider: Spider) -> None: - self.host_regex: re.Pattern[str] = self.get_host_regex(spider) + self._update_host_regex(spider) + + def _update_host_regex(self, spider: Spider) -> None: + allowed_domains = list(getattr(spider, "allowed_domains", None) or []) + if allowed_domains != self._allowed_domains: + self._allowed_domains = allowed_domains + self.host_regex = self.get_host_regex(spider) def request_scheduled(self, request: Request, spider: Spider) -> None: self.process_request(request) @@ -64,6 +72,7 @@ class OffsiteMiddleware: raise IgnoreRequest(f"Filtered offsite request to {domain!r}") def should_follow(self, request: Request, spider: Spider) -> bool: + self._update_host_regex(spider) regex = self.host_regex # hostname can be None for wrong urls (like javascript links) host = urlparse_cached(request).hostname or "" diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index cb17c2553..bab89814f 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -247,3 +247,50 @@ def test_ignore_request_reason(): IgnoreRequest, match=re.escape("Filtered offsite request to 'other.org'") ): mw.process_request(request) + + +class DomainSpider(Spider): + name = "a" + allowed_domains: list[str] + + +def test_dynamic_allowed_domains(): + crawler = get_crawler(DomainSpider) + spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"]) + crawler.spider = spider + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + + with pytest.raises(IgnoreRequest): + mw.process_request(Request("https://b.example")) + + spider.allowed_domains.append("b.example") + assert mw.process_request(Request("https://b.example")) is None + + spider.allowed_domains.remove("a.example") + with pytest.raises(IgnoreRequest): + mw.process_request(Request("https://a.example")) + + +def test_dynamic_allowed_domains_caching(): + calls = 0 + + class TrackingMiddleware(OffsiteMiddleware): + def get_host_regex(self, spider: Spider) -> re.Pattern[str]: + nonlocal calls + calls += 1 + return super().get_host_regex(spider) + + crawler = get_crawler(DomainSpider) + spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"]) + crawler.spider = spider + mw = TrackingMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + + for _ in range(3): + mw.process_request(Request("https://a.example")) + assert calls == 1 + + spider.allowed_domains.append("b.example") + assert mw.process_request(Request("https://b.example")) is None + assert calls == 2 From 0b2d220197fb7d9b274738cd812cb7d953f85c49 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:19:50 +0200 Subject: [PATCH 071/111] Improve docs for multi-spider runs (#7907) --- docs/topics/practices.rst | 12 ++++++++++++ docs/topics/settings.rst | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index dfa1e21f6..aeaf322c2 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -458,6 +458,18 @@ finishes before starting the next one: should not have a different value per spider, and :ref:`pre-crawler settings ` cannot be defined per spider. +Every other setting applies to each crawler separately. This includes +concurrency and politeness settings, such as :setting:`CONCURRENT_REQUESTS`, +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`, and +:ref:`AutoThrottle ` also throttles each crawler +separately. When crawling simultaneously, divide those values by the number of +crawlers to keep the combined load on your hardware and on target websites +unchanged. + +Because of this, running the same spider several times in the same process +multiplies those limits instead of increasing crawling capacity. To crawl +faster, raise :setting:`CONCURRENT_REQUESTS` on a single crawler. + .. seealso:: :ref:`run-from-script`. .. skip: end diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 793d65f35..dc09fdeeb 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -754,6 +754,11 @@ Default: ``60`` Timeout for processing of DNS queries in seconds. Float is supported. +The timeout starts when the query is queued into the Twisted reactor thread +pool, not when it is sent. If that thread pool is saturated, queries can time +out before being sent, in which case increasing +:setting:`REACTOR_THREADPOOL_MAXSIZE` helps more than increasing this setting. + .. note:: This setting is only used by :class:`~scrapy.resolver.CachingThreadedResolver`. It has no effect when From 9e84112221133f6a36dac1e0e6072a39ddfb2fd2 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:25:14 +0200 Subject: [PATCH 072/111] Cover update_vars() in the shell docs (#7889) --- docs/topics/shell.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 6f7e67cf9..42c5bd169 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -144,6 +144,32 @@ Those objects are: - ``settings`` - the current :ref:`Scrapy settings ` +.. _shell-update-vars: + +Adding your own objects +----------------------- + +To define additional objects, or to run code every time a response is fetched, +write a :ref:`custom project command ` in a module called +``shell``, which overrides the :command:`shell` command, and override its +``update_vars`` method. It is called on start and after every ``fetch``, and it +receives the mapping of variable names to objects: + +.. code-block:: python + + from scrapy.commands.shell import Command as ShellCommand + + + class Command(ShellCommand): + def update_vars(self, vars): + from myproject.utils import parse_product + + vars["parse_product"] = parse_product + if vars["response"] is not None: + vars["product"] = parse_product(vars["response"]) + +``response`` is ``None`` when the shell is started without a URL. + Example of shell session ======================== From f123c7a1cc9974c85450f45bff6d18ac49056ed5 Mon Sep 17 00:00:00 2001 From: Mridankan Mandal Date: Sun, 9 Aug 2026 15:25:58 +0530 Subject: [PATCH 073/111] Simplify cmdline settings test (#7853) Signed-off-by: Mridankan Mandal --- tests/test_cmdline/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 98a85bc17..f6ebe5865 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,3 +1,4 @@ +import ast import json import os import pstats @@ -60,11 +61,8 @@ class TestCmdline: "-s", "EXTENSIONS=" + json.dumps(EXTENSIONS), ) - # XXX: There's gotta be a smarter way to do this... assert "..." not in settingsstr - for char in ("'", "<", ">"): - settingsstr = settingsstr.replace(char, '"') - settingsdict = json.loads(settingsstr) + settingsdict = ast.literal_eval(settingsstr) assert set(settingsdict.keys()) == set(EXTENSIONS.keys()) assert settingsdict[EXT_PATH] == 200 From cd7f422a1e477039ba40e96979b5f060030ac702 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:57:11 +0200 Subject: [PATCH 074/111] Use client.bucket() in GCSFeedStorage so object-level GCS permissions suffice (#7945) --- scrapy/extensions/feedexport.py | 2 +- tests/test_feedexport_storages.py | 4 ++-- tests/utils/cloud.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 118462bc9..fc2b2f43f 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -340,7 +340,7 @@ class GCSFeedStorage(BlockingFeedStorage): from google.cloud.storage import Client # noqa: PLC0415 client = Client(project=self.project_id) - bucket = client.get_bucket(self.bucket_name) + bucket = client.bucket(self.bucket_name) blob = bucket.blob(self.blob_name) blob.upload_from_file(file, predefined_acl=self.acl) finally: diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index 4d28872b7..6f9e33449 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -524,7 +524,7 @@ class TestGCSFeedStorage: f.seek.assert_called_once_with(0) m.assert_called_once_with(project=project_id) - client_mock.get_bucket.assert_called_once_with("mybucket") + client_mock.bucket.assert_called_once_with("mybucket") bucket_mock.blob.assert_called_once_with("export.csv") blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) f.close.assert_called_once_with() @@ -548,7 +548,7 @@ class TestGCSFeedStorage: f.seek.assert_called_once_with(0) m.assert_called_once_with(project=project_id) - client_mock.get_bucket.assert_called_once_with("mybucket") + client_mock.bucket.assert_called_once_with("mybucket") bucket_mock.blob.assert_called_once_with("export.csv") blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) f.close.assert_called_once_with() diff --git a/tests/utils/cloud.py b/tests/utils/cloud.py index 662e0b3ee..4e253fbdc 100644 --- a/tests/utils/cloud.py +++ b/tests/utils/cloud.py @@ -14,7 +14,6 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: bucket_mock = mock.create_autospec(Bucket) client_mock.bucket.return_value = bucket_mock - client_mock.get_bucket.return_value = bucket_mock blob_mock = mock.create_autospec(Blob) bucket_mock.blob.return_value = blob_mock From a18d58d7b50e22c8356aef3efaf50ae36e6b18a2 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 12:26:15 +0200 Subject: [PATCH 075/111] Document that process_spider_output receives a lazy result (#7939) --- docs/topics/spider-middleware.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index aa14f6801..78b211dac 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,6 +122,9 @@ one or more of these methods: This method is an :term:`asynchronous generator` called with the results from the spider after the spider has processed the response. + *result* is lazy: a generator callback runs as *result* is iterated, so + code that runs before that iteration runs before the callback body. + .. seealso:: :ref:`universal-spider-middleware`. :param response: the response which generated this output from the From 63d5ce6272a7a6596672d6187a0c56c819782deb Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 13:52:32 +0200 Subject: [PATCH 076/111] Fix cookie lookup for dotless hosts and IP addresses (#7900) --- scrapy/http/cookies.py | 4 ++-- tests/test_downloadermiddleware_cookies.py | 24 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 8edeae01c..555d930e6 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -54,9 +54,9 @@ class CookieJar: if not IPV4_RE.search(req_host): hosts = potential_domain_matches(req_host) if "." not in req_host: - hosts.append(req_host + ".local") + hosts += potential_domain_matches(req_host + ".local") else: - hosts = [req_host] + hosts = [req_host, "." + req_host] cookies = [] for host in hosts: diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 8d999d952..e4b66fe10 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -345,6 +345,30 @@ class TestCookiesMiddleware: assert "Cookie" in request.headers assert request.headers["Cookie"] == b"currencyCookie=USD" + @pytest.mark.parametrize( + ("url", "domain"), + [ + ("http://example-host/", "example-host.local"), + ("http://127.0.0.1/", "127.0.0.1"), + pytest.param( + "http://example-host/", + "example-host", + marks=pytest.mark.xfail( + reason=( + "http.cookiejar accepts a dotless domain for a dotless " + "host but never returns the resulting cookie" + ) + ), + ), + ], + ) + def test_explicit_local_domain(self, url: str, domain: str) -> None: + request = Request( + url, cookies=[{"name": "currencyCookie", "value": "USD", "domain": domain}] + ) + assert self.mw.process_request(request) is None + assert request.headers.get("Cookie") == b"currencyCookie=USD" + @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_from_default_request_headers_middleware(self): DEFAULT_REQUEST_HEADERS = {"Cookie": "default=value; asdf=qwerty"} From 7c797968a31ecacc24e17b81e57aad44de803389 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 13:53:33 +0200 Subject: [PATCH 077/111] Docs: clarify the handling of exceptions raised in errbacks (#7898) --- docs/topics/request-response.rst | 4 ++++ docs/topics/spider-middleware.rst | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 75158440b..f810146e4 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -770,6 +770,10 @@ is raise while processing it. It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can be used to track connection establishment timeouts, DNS errors etc. +If an errback raises an exception, Scrapy logs it and sends the +:signal:`spider_error` signal, unless the exception is the one that the errback +received, which Scrapy logs as a download error instead. + Here's an example spider logging all errors and catching some specific errors if needed: diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 78b211dac..db85906c1 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -145,8 +145,9 @@ one or more of these methods: .. method:: process_spider_exception(response, exception) - This method is called when a spider or :meth:`process_spider_output` - method (from a previous spider middleware) raises an exception. + This method is called when a spider callback or a + :meth:`process_spider_output` method (from a previous spider + middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an iterable of :class:`~scrapy.Request` or :ref:`item ` From 59ce27afddd4c30e72c4412c9b2b7a41183249a1 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 13:55:02 +0200 Subject: [PATCH 078/111] Send the bytes_received and headers_received signals over HTTP/2 (#7896) --- docs/topics/download-handlers.rst | 3 - scrapy/core/downloader/handlers/http2.py | 2 +- scrapy/core/http2/agent.py | 10 +-- scrapy/core/http2/protocol.py | 18 +++-- scrapy/core/http2/stream.py | 74 +++++++++++++++---- .../test_downloader_handler_twisted_http2.py | 12 --- tests/test_http2_client_protocol.py | 4 +- 7 files changed, 76 insertions(+), 47 deletions(-) diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index e0501c169..94e75ab6f 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -203,9 +203,6 @@ Other limitations: - IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. -- No support for the :signal:`bytes_received` and :signal:`headers_received` - signals. - Known limitations of the HTTP/2 support: - No support for HTTP/2 Cleartext (h2c), since no major browser supports diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index f60c58d1b..9b3d4fbd4 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -40,7 +40,7 @@ class H2DownloadHandler(BaseDownloadHandler): from twisted.internet import reactor - self._pool = H2ConnectionPool(reactor, crawler.settings) + self._pool = H2ConnectionPool(reactor, crawler) self._context_factory = _load_context_factory_from_settings(crawler) self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS") diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index aa55e29a0..042557208 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -21,8 +21,8 @@ if TYPE_CHECKING: from twisted.internet.base import ReactorBase from twisted.internet.endpoints import HostnameEndpoint + from scrapy.crawler import Crawler from scrapy.http import Request, Response - from scrapy.settings import Settings from scrapy.spiders import Spider @@ -30,9 +30,9 @@ ConnectionKeyT = tuple[bytes, bytes, int] class H2ConnectionPool: - def __init__(self, reactor: ReactorBase, settings: Settings) -> None: + def __init__(self, reactor: ReactorBase, crawler: Crawler) -> None: self._reactor = reactor - self.settings = settings + self._crawler = crawler # Store a dictionary which is used to get the respective # H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port) @@ -43,7 +43,7 @@ class H2ConnectionPool: ConnectionKeyT, deque[Deferred[H2ClientProtocol]] ] = {} - self._tls_verbose_logging: bool = settings.getbool( + self._tls_verbose_logging: bool = crawler.settings.getbool( "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" ) @@ -77,7 +77,7 @@ class H2ConnectionPool: factory = H2ClientFactory( uri, - self.settings, + self._crawler, conn_lost_deferred, tls_verbose_logging=self._tls_verbose_logging, ) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 7136e829e..2d59aba31 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -44,7 +44,7 @@ if TYPE_CHECKING: from twisted.python.failure import Failure from twisted.web.client import URI - from scrapy.settings import Settings + from scrapy.crawler import Crawler from scrapy.spiders import Spider @@ -90,7 +90,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def __init__( self, uri: URI, - settings: Settings, + crawler: Crawler, conn_lost_deferred: Deferred[list[BaseException]], *, tls_verbose_logging: bool = False, @@ -100,11 +100,12 @@ class H2ClientProtocol(Protocol, TimeoutMixin): uri -- URI of the base url to which HTTP/2 Connection will be made. uri is used to verify that incoming client requests have correct base URL. - settings -- Scrapy project settings + crawler -- The crawler the requests belong to conn_lost_deferred -- Deferred that fires with the list of underlying exceptions to notify that connection was lost tls_verbose_logging -- Whether to log TLS details """ + self._crawler: Crawler = crawler self._conn_lost_deferred: Deferred[list[BaseException]] = conn_lost_deferred self._tls_verbose_logging: bool = tls_verbose_logging @@ -140,8 +141,8 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # Both ip_address and uri are used by the Stream before # initiating the request to verify that the base address # Variables taken from Project Settings - "default_download_maxsize": settings.getint("DOWNLOAD_MAXSIZE"), - "default_download_warnsize": settings.getint("DOWNLOAD_WARNSIZE"), + "default_download_maxsize": crawler.settings.getint("DOWNLOAD_MAXSIZE"), + "default_download_warnsize": crawler.settings.getint("DOWNLOAD_WARNSIZE"), # Counter to keep track of opened streams. This counter # is used to make sure that not more than MAX_CONCURRENT_STREAMS # streams are opened which leads to ProtocolError @@ -208,6 +209,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): stream_id=next(self._stream_id_generator), request=request, protocol=self, + crawler=self._crawler, download_maxsize=getattr( spider, "download_maxsize", self.metadata["default_download_maxsize"] ), @@ -461,20 +463,20 @@ class H2ClientFactory(Factory): def __init__( self, uri: URI, - settings: Settings, + crawler: Crawler, conn_lost_deferred: Deferred[list[BaseException]], *, tls_verbose_logging: bool = False, ) -> None: self.uri = uri - self.settings = settings + self.crawler = crawler self.conn_lost_deferred = conn_lost_deferred self.tls_verbose_logging = tls_verbose_logging def buildProtocol(self, addr: IAddress) -> H2ClientProtocol: return H2ClientProtocol( self.uri, - self.settings, + self.crawler, self.conn_lost_deferred, tls_verbose_logging=self.tls_verbose_logging, ) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index c6226bbca..4fc300d90 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from contextlib import suppress from enum import Enum from io import BytesIO from typing import TYPE_CHECKING, Any @@ -12,9 +13,11 @@ from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from scrapy.exceptions import DownloadCancelledError +from scrapy import signals +from scrapy.exceptions import DownloadCancelledError, StopDownload from scrapy.http.headers import Headers from scrapy.utils._download_handlers import ( + check_stop_download, get_maxsize_msg, get_warnsize_msg, make_response, @@ -25,6 +28,7 @@ if TYPE_CHECKING: from collections.abc import Sequence from scrapy.core.http2.protocol import H2ClientProtocol + from scrapy.crawler import Crawler from scrapy.http import Request, Response @@ -82,6 +86,9 @@ class StreamCloseReason(Enum): # Actual response body size is more than allowed limit MAXSIZE_EXCEEDED_ACTUAL = 8 + # A signal handler raised StopDownload + STOP_DOWNLOAD = 9 + class Stream: """Represents a single HTTP/2 Stream. @@ -99,6 +106,7 @@ class Stream: stream_id: int, request: Request, protocol: H2ClientProtocol, + crawler: Crawler, download_maxsize: int = 0, download_warnsize: int = 0, ) -> None: @@ -107,10 +115,13 @@ class Stream: stream_id -- Unique identifier for the stream within a single HTTP/2 connection request -- The HTTP request associated to the stream protocol -- Parent H2ClientProtocol instance + crawler -- The crawler the request belongs to """ self.stream_id: int = stream_id self._request: Request = request self._protocol: H2ClientProtocol = protocol + self._crawler: Crawler = crawler + self._stop_download: StopDownload | None = None self._download_maxsize = self._request.meta.get( "download_maxsize", download_maxsize @@ -338,6 +349,13 @@ class Stream: self._response["body"].write(data) self._response["flow_controlled_size"] += flow_controlled_length + if stop_download := check_stop_download( + signals.bytes_received, self._crawler, self._request, data=data + ): + self._stop_download = stop_download + self.reset_stream(StreamCloseReason.STOP_DOWNLOAD) + return + # We check maxsize here in case the Content-Length header was not received if ( self._download_maxsize @@ -369,8 +387,20 @@ class Stream: else: self._response["headers"].appendlist(name, value) - # Check if we exceed the allowed max data size which can be received expected_size = int(self._response["headers"].get(b"Content-Length", -1)) + + if stop_download := check_stop_download( + signals.headers_received, + self._crawler, + self._request, + headers=self._response["headers"], + body_length=expected_size if expected_size >= 0 else None, + ): + self._stop_download = stop_download + self.reset_stream(StreamCloseReason.STOP_DOWNLOAD) + return + + # Check if we exceed the allowed max data size which can be received if self._download_maxsize and expected_size > self._download_maxsize: self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) return @@ -387,11 +417,18 @@ class Stream: if self.metadata["stream_closed_local"]: raise StreamClosedError(self.stream_id) - # Clear buffer earlier to avoid keeping data in memory for a long time - self._response["body"].truncate(0) + # The data received so far is the body of the response built for a + # stopped download, otherwise the buffer is cleared early to avoid + # keeping data in memory for a long time + if reason is not StreamCloseReason.STOP_DOWNLOAD: + self._response["body"].truncate(0) self.metadata["stream_closed_local"] = True - self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) + # The remote peer may have ended the stream already, e.g. because the + # whole response arrived within the data that triggered this reset, in + # which case there is nothing left to reset + with suppress(StreamClosedError): + self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) self.close(reason) def close( @@ -444,7 +481,7 @@ class Stream: logger.error(error_msg) self._deferred_response.errback(DownloadCancelledError(error_msg)) - elif reason is StreamCloseReason.ENDED: + elif reason in {StreamCloseReason.ENDED, StreamCloseReason.STOP_DOWNLOAD}: self._fire_response_deferred() # Stream was abruptly ended here @@ -495,13 +532,18 @@ class Stream: and fires the response deferred callback with the generated response instance""" - response = make_response( - url=self._request.url, - status=self._response["status"], - headers=self._response["headers"], - body=self._response["body"].getvalue(), - certificate=self._protocol.metadata["certificate"], - ip_address=self._protocol.metadata["ip_address"], - protocol="h2", - ) - self._deferred_response.callback(response) + try: + response = make_response( + url=self._request.url, + status=self._response["status"], + headers=self._response["headers"], + body=self._response["body"].getvalue(), + certificate=self._protocol.metadata["certificate"], + ip_address=self._protocol.metadata["ip_address"], + protocol="h2", + stop_download=self._stop_download, + ) + except StopDownload as exc: + self._deferred_response.errback(exc) + else: + self._deferred_response.callback(response) diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 449f2d635..9d4e161f3 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -186,18 +186,6 @@ class TestHttp2TLSVersion(H2DownloadHandlerMixin, TestHttpsTLSVersionBase): class TestHttp2WithCrawler(H2DownloadHandlerMixin, TestHttpWithCrawlerBase): is_secure = True - def test_bytes_received_stop_download_callback(self) -> None: # type: ignore[override] - pytest.skip("bytes_received support is not implemented") - - def test_bytes_received_stop_download_errback(self) -> None: # type: ignore[override] - pytest.skip("bytes_received support is not implemented") - - def test_headers_received_stop_download_callback(self) -> None: # type: ignore[override] - pytest.skip("headers_received support is not implemented") - - def test_headers_received_stop_download_errback(self) -> None: # type: ignore[override] - pytest.skip("headers_received support is not implemented") - @pytest.mark.skip(reason="Proxy support is not implemented yet") class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index b8586d1ca..431b7a458 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -23,13 +23,13 @@ from twisted.web.static import File from scrapy.exceptions import DownloadCancelledError, DownloadTimeoutError from scrapy.http import JsonRequest, Request, Response -from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.defer import ( deferred_f_from_coro_f, deferred_from_coro, maybe_deferred_to_future, ) +from scrapy.utils.test import get_crawler from tests.mockserver.http_resources import LeafResource, Status, put_child from tests.mockserver.utils import ssl_context_factory @@ -250,7 +250,7 @@ class TestHttps2ClientProtocol: acceptableProtocols=[b"h2"], ) uri = URI.fromBytes(bytes(self.get_url(server_port, "/"), "utf-8")) - h2_client_factory = H2ClientFactory(uri, Settings(), Deferred()) + h2_client_factory = H2ClientFactory(uri, get_crawler(), Deferred()) client_endpoint = SSL4ClientEndpoint( reactor, self.host, server_port, client_options ) From 050a8cf159a6b0c0c7d4ba08b98d957c0ab3416d Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 13:57:08 +0200 Subject: [PATCH 079/111] Improve the docs about delaying start request iteration (#7883) --- docs/topics/broad-crawls.rst | 5 +++-- docs/topics/signals.rst | 9 +++++++++ docs/topics/spiders.rst | 12 ++++++------ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index cace1f883..d6b9fd6f9 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -182,8 +182,9 @@ Be mindful of memory leaks ========================== If your broad crawl shows a high memory usage, in addition to :ref:`crawling in -BFO order ` and :ref:`lowering concurrency -` you should :ref:`debug your memory leaks +BFO order `, :ref:`lowering concurrency +` and :ref:`delaying start request iteration +` you should :ref:`debug your memory leaks `. diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index ceea2f7c0..f060710a4 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -158,6 +158,15 @@ scheduler_empty See :ref:`start-requests-lazy` for an example. + .. warning:: Only wait for this signal from + :meth:`~scrapy.Spider.start`. While no request can be sent, e.g. while + the responses being parsed exceed + :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE`, the engine does not ask the + scheduler for requests, and hence this signal is not sent. So waiting + for it from a :ref:`callback ` can hang the crawl, + because the response being parsed is itself one of the responses that + may be blocking requests. + This signal does not support :ref:`asynchronous handlers `. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index f2cfeb712..e68c208b7 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -396,8 +396,12 @@ Start requests Delaying start request iteration -------------------------------- -You can override the :meth:`~scrapy.Spider.start` method as follows to pause -its iteration whenever there are scheduled requests: +Scrapy iterates :meth:`~scrapy.Spider.start` as fast as it yields, so all start +requests reach the scheduler early in the crawl, however many they are. To +minimize the number of requests in the scheduler at any given time, and with it +resource usage (memory, or disk when using :setting:`JOBDIR`), override +:meth:`~scrapy.Spider.start` to pause its iteration whenever there are +scheduled requests: .. code-block:: python @@ -407,10 +411,6 @@ its iteration whenever there are scheduled requests: await self.crawler.signals.wait_for(signals.scheduler_empty) yield item_or_request -This can help minimize the number of requests in the scheduler at any given -time, to minimize resource usage (memory or disk, depending on -:setting:`JOBDIR`). - .. _builtin-spiders: Generic Spiders From 18ed0c0f7cea0fc23e1f837b72059889191e735f Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 13:58:09 +0200 Subject: [PATCH 080/111] Fall back to the response encoding in TextResponse.json() (#7897) --- docs/topics/request-response.rst | 3 --- scrapy/http/response/text.py | 16 ++++++++++++++-- tests/test_http_response_text.py | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f810146e4..a177d1ad5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -1432,9 +1432,6 @@ TextResponse objects .. automethod:: TextResponse.json() - Returns a Python object from deserialized JSON document. - The result is cached after the first call. - .. method:: TextResponse.urljoin(url) Constructs an absolute url by combining the Response's base url with diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index d01e23e47..64251780a 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -84,9 +84,21 @@ class TextResponse(Response): ) def json(self) -> Any: - """Deserialize a JSON document to a Python object.""" + """Deserialize a JSON document to a Python object. + + .. versionchanged:: VERSION + Bodies that cannot be decoded as UTF-8, UTF-16 or UTF-32, as the + JSON specification requires, are now decoded using + :attr:`TextResponse.encoding` instead of raising + :exc:`UnicodeDecodeError`. + + The result is cached after the first call. + """ if self._cached_decoded_json is _NONE: - self._cached_decoded_json = json.loads(self.body) + try: + self._cached_decoded_json = json.loads(self.body) + except UnicodeDecodeError: + self._cached_decoded_json = json.loads(self.text) return self._cached_decoded_json @property diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index efa63e049..f705dbcee 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -481,6 +481,22 @@ class TestTextResponse(TestResponseBase): ): text_response.json() + def test_json_response_non_utf8(self): + response = self.response_class( + "http://www.example.com", + body='{"message": "café"}'.encode("cp1252"), + headers={"Content-Type": "application/json"}, + ) + assert response.json() == {"message": "café"} + + def test_json_response_wrong_charset(self): + response = self.response_class( + "http://www.example.com", + body='{"message": "café"}'.encode(), + headers={"Content-Type": "application/json; charset=iso-8859-1"}, + ) + assert response.json() == {"message": "café"} + def test_cache_json_response(self): json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""] for json_body in json_valid_bodies: From a6c017c2ca2b8a9bca8b46b692846f492fe77a00 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 13:59:22 +0200 Subject: [PATCH 081/111] Advise setting an identifying user agent (#7890) --- docs/intro/tutorial.rst | 5 ++++ docs/topics/practices.rst | 51 +++++++++++++++++++++++---------------- docs/topics/settings.rst | 5 ++++ 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index eaf492c95..efade47e6 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -72,6 +72,11 @@ This will create a ``tutorial`` directory with the following contents:: spiders/ # a directory where you'll later put your spiders __init__.py +Before crawling anything, open ``settings.py`` and uncomment the +:setting:`USER_AGENT` line to identify yourself, e.g. a project name plus a URL +or an email address. Website owners who take issue with your crawler can then +ask you to adjust it, rather than block it. + Our first Spider ================ diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index aeaf322c2..971bb9106 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -530,32 +530,41 @@ modules by separating them with commas. Avoiding getting banned ======================= -Some websites implement certain measures to prevent bots from crawling them, -with varying degrees of sophistication. Getting around those measures can be -difficult and tricky, and may sometimes require special infrastructure. Please -consider contacting `commercial support`_ if in doubt. +Websites tell regular visitors and crawlers apart by how their traffic looks: +the headers it carries, how fast it arrives, how many requests come from the +same place. Traffic that stands out can be blocked even when the crawling +itself would be welcome. -Here are some tips to keep in mind when dealing with these kinds of sites: +Where the website allows crawling, the most effective thing you can do is make +yourself known: set :setting:`USER_AGENT` to a value that identifies you and +lets its owners reach you, so that they can ask you to adjust your crawler +rather than block it. -* rotate your user agent from a pool of well-known ones from browsers (Google - around to get a list of them) -* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use - cookies to spot bot behaviour -* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting. -* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites - directly -* use a pool of rotating IPs. For example, the free `Tor project`_ or paid +Where that is not enough, the following make your traffic resemble that of a +regular visitor: + +* rotate your user agent among those of common browsers, so that your requests + do not all look alike (search the web for an up-to-date list) +* disable cookies (see :setting:`COOKIES_ENABLED`), so that a session + identifier does not tie all your requests together +* space out your requests, 2 seconds apart or more, with the + :setting:`DOWNLOAD_DELAY` setting, to keep your pace closer to that of a + person browsing +* where possible, read pages from `Common Crawl`_, which sends no traffic to + the website at all +* spread your requests over a pool of IP addresses, so that none of them + accounts for your whole crawl. For example, the free `Tor project`_ or paid services like `ProxyMesh`_. -* for HTTPS websites, if blocking appears related to TLS behavior, consider - adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and - :setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond - differently depending on the TLS method used by the client. -* use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy - plugin `__ and additional +* match the TLS behavior of a browser: some websites respond differently + depending on the TLS version of the client, which you can adjust with the + :setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION` + settings. +* let a service take care of all of the above, such as `Zyte API`_, which + provides a `Scrapy plugin + `__ and additional features, like `AI web scraping `__ -If you are still unable to prevent your bot getting banned, consider contacting -`commercial support`_. +If your crawler still gets blocked, consider contacting `commercial support`_. .. _static-analysis: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index dc09fdeeb..ec222d999 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2338,6 +2338,11 @@ also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and there is no overriding User-Agent header specified for the request. +Set it to a value that identifies you, including a URL or an email address +where website owners can reach you, e.g. ``"MyProject +(+https://example.com/bot)"``, so that they can ask you to adjust your crawler +rather than block it. + .. setting:: WARN_ON_GENERATOR_RETURN_VALUE WARN_ON_GENERATOR_RETURN_VALUE From 95fe7acc86f83643bfa98d5a0da97b8457a61774 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 18:14:20 +0200 Subject: [PATCH 082/111] Warn only once per request about the download warn size (#7963) --- scrapy/core/downloader/handlers/http11.py | 7 +++++-- tests/utils/bases/download_handlers_http.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 17dca5acb..3b5c08666 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -548,7 +548,8 @@ class _ScrapyAgent: txresponse._transport._producer.abortConnection() raise DownloadCancelledError(warning_msg) - if warnsize and expected_size > warnsize: + reached_warnsize = bool(warnsize and expected_size > warnsize) + if reached_warnsize: logger.warning( get_warnsize_msg(expected_size, warnsize, request, expected=True) ) @@ -561,6 +562,7 @@ class _ScrapyAgent: request=request, maxsize=maxsize, warnsize=warnsize, + reached_warnsize=reached_warnsize, fail_on_dataloss=fail_on_dataloss, crawler=self._crawler, tls_verbose_logging=self._tls_verbose_logging, @@ -625,6 +627,7 @@ class _ResponseReader(Protocol): fail_on_dataloss: bool, crawler: Crawler, *, + reached_warnsize: bool = False, tls_verbose_logging: bool = False, ): self._finished: Deferred[_ResultT] = finished @@ -634,7 +637,7 @@ class _ResponseReader(Protocol): self._maxsize: int = maxsize self._warnsize: int = warnsize self._fail_on_dataloss: bool = fail_on_dataloss - self._reached_warnsize: bool = False + self._reached_warnsize: bool = reached_warnsize self._bytes_received: int = 0 self._certificate: ssl.Certificate | None = None self._ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py index e44f9bcb8..65244b938 100644 --- a/tests/utils/bases/download_handlers_http.py +++ b/tests/utils/bases/download_handlers_http.py @@ -628,6 +628,7 @@ class TestHttpBase(ABC): "Expected to receive 5 bytes which is larger than download warn size (4)" in caplog.text ) + assert caplog.text.count("download warn size (4)") == 1 @coroutine_test async def test_download_with_warnsize_no_content_length( From a0d1ad20013cd24da5aeba2c7742a2719d83f599 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 18:15:27 +0200 Subject: [PATCH 083/111] Document that follow_all requests share meta and cb_kwargs values (#7962) --- scrapy/http/response/__init__.py | 5 +++++ scrapy/http/response/text.py | 3 +++ scrapy/spidermiddlewares/metacopy.py | 3 +-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f1db11488..f91cb2095 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -305,6 +305,11 @@ class Response(object_ref): :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all` method which supports selectors in addition to absolute/relative URLs and Link objects. + + .. caution:: Every returned request gets its own *meta* and + *cb_kwargs* dictionaries, but the values within them are shared. + Mutating one of those values, e.g. appending to a list, affects + all the returned requests. """ if not hasattr(urls, "__iter__"): raise TypeError("'urls' argument must be an iterable") diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 64251780a..a8e452f7e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -276,6 +276,9 @@ class TextResponse(Response): using the ``css`` or ``xpath`` parameters, this method will not produce requests for selectors from which links cannot be obtained (for instance, anchor tags without an ``href`` attribute) + + .. seealso:: :meth:`.Response.follow_all`, for a caution about mutable + *meta* and *cb_kwargs* values. """ arguments = [x for x in (urls, css, xpath) if x is not None] if len(arguments) != 1: diff --git a/scrapy/spidermiddlewares/metacopy.py b/scrapy/spidermiddlewares/metacopy.py index aa5a120c6..2bef974d6 100644 --- a/scrapy/spidermiddlewares/metacopy.py +++ b/scrapy/spidermiddlewares/metacopy.py @@ -15,8 +15,7 @@ logger = logging.getLogger(__name__) class MetaCopyDetectionMiddleware(BaseSpiderMiddleware): """Warn when a spider yields a request with internal meta keys that should - not be copied from response.meta, or when two requests share the same meta - dict object. + not be copied from response.meta. Each warning is emitted at most once per crawl. """ From c67bb8ac47c0af3ee78ef7f55642a85c701c2565 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 18:15:37 +0200 Subject: [PATCH 084/111] Document BROWSER=wslview for open_in_browser under WSL (#7965) --- scrapy/utils/response.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 7747a7b9b..b3622c159 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -90,6 +90,10 @@ def open_in_browser( def parse_details(self, response): if "item name" not in response.text: open_in_browser(response) + + On the Windows Subsystem for Linux, set the ``BROWSER`` environment + variable to `wslview `_ to open the + response in a Windows browser, which cannot read Linux paths otherwise. """ # circular imports from scrapy.http import HtmlResponse, TextResponse # noqa: PLC0415 From 8e94df65262f9d4d2d016bd3bc173c90c766c390 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 18:18:33 +0200 Subject: [PATCH 085/111] Time out hanging tests in the VCS dependencies job (#7960) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Time out hanging tests in the VCS dependencies job * Do not test pydispatcher’s latest commit --- .github/workflows/tests-vcs-deps.yml | 5 ++++- tox.ini | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-vcs-deps.yml b/.github/workflows/tests-vcs-deps.yml index bf867dba7..93f262b62 100644 --- a/.github/workflows/tests-vcs-deps.yml +++ b/.github/workflows/tests-vcs-deps.yml @@ -16,8 +16,11 @@ jobs: tests: name: tests runs-on: ubuntu-latest + timeout-minutes: 30 env: - PYTEST_ADDOPTS: -n auto --no-cov + # A development branch of a dependency can make a test hang forever, so + # tests get a time limit here that they do not need elsewhere. + PYTEST_ADDOPTS: -n auto --no-cov --timeout=120 TOXENV: vcs-deps UV_PYTHON_PREFERENCE: only-system steps: diff --git a/tox.ini b/tox.ini index 94c04f75f..ab53111b8 100644 --- a/tox.ini +++ b/tox.ini @@ -44,6 +44,7 @@ deps = pygments pytest pytest-cov >= 7.0.0 + pytest-timeout pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 pytest-twisted >= 1.14.3 @@ -214,7 +215,8 @@ deps = # # Pillow and uvloop build from source, and need the libjpeg headers and # autotools respectively. robotexclusionrulesparser has no public repository, -# so it stays at its latest release. +# and PyDispatcher has seen no commit since 2023-10-23, so they stay at their +# latest release. commands_pre = uv pip install --python {envpython} --no-deps --reinstall \ git+https://github.com/twisted/twisted \ @@ -235,7 +237,6 @@ commands_pre = git+https://github.com/john-kurkowski/tldextract \ git+https://github.com/scrapy/w3lib \ git+https://github.com/zopefoundation/zope.interface \ - git+https://github.com/mcfletch/pydispatcher \ git+https://github.com/boto/boto3 \ git+https://github.com/bpython/bpython \ git+https://github.com/google/brotli \ From 5427080f4892fb8f33cc159fdcd6f8ce8d4c140e Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 18:37:24 +0200 Subject: [PATCH 086/111] Create a page on optimization (#7938) --- docs/conf.py | 5 + docs/index.rst | 6 +- docs/requirements.in | 1 + docs/requirements.txt | 3 + docs/topics/broad-crawls.rst | 195 ------------------- docs/topics/optimize.rst | 353 +++++++++++++++++++++++++++++++++++ docs/topics/settings.rst | 1 + 7 files changed, 366 insertions(+), 198 deletions(-) delete mode 100644 docs/topics/broad-crawls.rst create mode 100644 docs/topics/optimize.rst diff --git a/docs/conf.py b/docs/conf.py index 1b41adaad..ad55231bc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,9 +31,14 @@ extensions = [ "sphinx_scrapy", "scrapyfixautodoc", # Must be after "sphinx.ext.autodoc" "sphinx.ext.coverage", + "sphinx_reredirects", "sphinx_rtd_dark_mode", ] +redirects = { + "topics/broad-crawls": "optimize.html#broad-crawls", +} + templates_path = ["_templates"] exclude_patterns = ["build", "Thumbs.db", ".DS_Store"] diff --git a/docs/index.rst b/docs/index.rst index 688cab81b..d45b2c208 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -152,7 +152,7 @@ Solving specific problems topics/contracts topics/practices topics/security - topics/broad-crawls + topics/optimize topics/developer-tools topics/dynamic-content topics/leaks @@ -180,8 +180,8 @@ Solving specific problems Understand the security implications of Scrapy defaults and how to harden them. -:doc:`topics/broad-crawls` - Tune Scrapy for crawling a lot domains in parallel. +:doc:`topics/optimize` + Find the bottleneck of your crawls and learn how to address it. :doc:`topics/developer-tools` Learn how to scrape with your browser's developer tools. diff --git a/docs/requirements.in b/docs/requirements.in index 3783dd1dc..77514642a 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -3,6 +3,7 @@ pydantic scrapy-spider-metadata sphinx sphinx-notfound-page +sphinx-reredirects sphinx-rtd-theme sphinx-rtd-dark-mode sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10 diff --git a/docs/requirements.txt b/docs/requirements.txt index 0f5969401..b0a6d0b04 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -134,6 +134,7 @@ sphinx==9.1.0 # sphinx-llms-txt # sphinx-markdown-builder # sphinx-notfound-page + # sphinx-reredirects # sphinx-rtd-theme # sphinx-scrapy # sphinxcontrib-jquery @@ -147,6 +148,8 @@ sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builde # via sphinx-scrapy sphinx-notfound-page==1.1.0 # via -r docs/requirements.in +sphinx-reredirects==1.1.0 + # via -r docs/requirements.in sphinx-rtd-dark-mode==1.3.0 # via -r docs/requirements.in sphinx-rtd-theme==3.1.0 diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst deleted file mode 100644 index d6b9fd6f9..000000000 --- a/docs/topics/broad-crawls.rst +++ /dev/null @@ -1,195 +0,0 @@ -.. _topics-broad-crawls: - -============ -Broad Crawls -============ - -Scrapy defaults are optimized for crawling specific sites. These sites are -often handled by a single Scrapy spider, although this is not necessary or -required (for example, there are generic spiders that handle any given site -thrown at them). - -In addition to this "focused crawl", there is another common type of crawling -which covers a large (potentially unlimited) number of domains, and is only -limited by time or other arbitrary constraint, rather than stopping when the -domain was crawled to completion or when there are no more requests to perform. -These are called "broad crawls" and is the typical crawlers employed by search -engines. - -These are some common properties often found in broad crawls: - -* they crawl many domains (often, unbounded) instead of a specific set of sites - -* they don't necessarily crawl domains to completion, because it would be - impractical (or impossible) to do so, and instead limit the crawl by time or - number of pages crawled - -* they are simpler in logic (as opposed to very complex spiders with many - extraction rules) because data is often post-processed in a separate stage - -* they crawl many domains concurrently, which allows them to achieve faster - crawl speeds by not being limited by any particular site constraint (each site - is crawled slowly to respect politeness, but many sites are crawled in - parallel) - -As said above, Scrapy default settings are optimized for focused crawls, not -broad crawls. However, due to its asynchronous architecture, Scrapy is very -well suited for performing fast broad crawls. This page summarizes some things -you need to keep in mind when using Scrapy for doing broad crawls, along with -concrete suggestions of Scrapy settings to tune in order to achieve an -efficient broad crawl. - -.. _broad-crawls-scheduler-priority-queue: - -.. _broad-crawls-concurrency: - -Increase concurrency -==================== - -Concurrency is the number of requests that are processed in parallel. There is -a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that -can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`). - -The default global concurrency limit in Scrapy is not suitable for crawling -many different domains in parallel, so you will want to increase it. How much -to increase it will depend on how much CPU and memory your crawler will have -available. - -A good starting point is ``100``: - -.. code-block:: python - - CONCURRENT_REQUESTS = 100 - -But the best way to find out is by doing some trials and identifying at what -concurrency your Scrapy process gets CPU bounded. For optimum performance, you -should pick a concurrency where CPU usage is at 80-90%. - -Increasing concurrency also increases memory usage. If memory usage is a -concern, you might need to lower your global concurrency limit accordingly. - - -Increase Twisted IO thread pool maximum size -============================================ - -Currently Scrapy does DNS resolution in a blocking way with usage of thread -pool. With higher concurrency levels the crawling could be slow or even fail -hitting DNS resolver timeouts. Possible solution to increase the number of -threads handling DNS queries. The DNS queue will be processed faster speeding -up establishing of connection and crawling overall. - -To increase maximum thread pool size use: - -.. code-block:: python - - REACTOR_THREADPOOL_MAXSIZE = 20 - -Setup your own DNS -================== - -If you have multiple crawling processes and single central DNS, it can act -like DoS attack on the DNS server resulting to slow down of entire network or -even blocking your machines. To avoid this setup your own DNS server with -local cache and upstream to some large DNS like OpenDNS or Verizon. - -Reduce log level -================ - -When doing broad crawls you are often only interested in the crawl rates you -get and any errors found. These stats are reported by Scrapy when using the -``INFO`` log level. In order to save CPU (and log storage requirements) you -should not use ``DEBUG`` log level when performing large broad crawls in -production. Using ``DEBUG`` level when developing your (broad) crawler may be -fine though. - -To set the log level use: - -.. code-block:: python - - LOG_LEVEL = "INFO" - -Disable cookies -=============== - -Disable cookies unless you *really* need. Cookies are often not needed when -doing broad crawls (search engine crawlers ignore them), and they improve -performance by saving some CPU cycles and reducing the memory footprint of your -Scrapy crawler. - -To disable cookies use: - -.. code-block:: python - - COOKIES_ENABLED = False - -Disable retries -=============== - -Retrying failed HTTP requests can slow down the crawls substantially, especially -when sites causes are very slow (or fail) to respond, thus causing a timeout -error which gets retried many times, unnecessarily, preventing crawler capacity -to be reused for other domains. - -To disable retries use: - -.. code-block:: python - - RETRY_ENABLED = False - -Reduce download timeout -======================= - -Unless you are crawling from a very slow connection (which shouldn't be the -case for broad crawls) reduce the download timeout so that stuck requests are -discarded quickly and free up capacity to process the next ones. - -To reduce the download timeout use: - -.. code-block:: python - - DOWNLOAD_TIMEOUT = 15 - -Disable redirects -================= - -Consider disabling redirects, unless you are interested in following them. When -doing broad crawls it's common to save redirects and resolve them when -revisiting the site at a later crawl. This also help to keep the number of -request constant per crawl batch, otherwise redirect loops may cause the -crawler to dedicate too many resources on any specific domain. - -To disable redirects use: - -.. code-block:: python - - REDIRECT_ENABLED = False - -.. _broad-crawls-bfo: - -Crawl in BFO order -================== - -:ref:`Scrapy crawls in DFO order by default `. - -In broad crawls, however, page crawling tends to be faster than page -processing. As a result, unprocessed early requests stay in memory until the -final depth is reached, which can significantly increase memory usage. - -:ref:`Crawl in BFO order ` instead to save memory. - - -Be mindful of memory leaks -========================== - -If your broad crawl shows a high memory usage, in addition to :ref:`crawling in -BFO order `, :ref:`lowering concurrency -` and :ref:`delaying start request iteration -` you should :ref:`debug your memory leaks -`. - - -Install a specific Twisted reactor -================================== - -If the crawl is exceeding the system's capabilities, you might want to try -installing a specific Twisted reactor, via the :setting:`TWISTED_REACTOR` setting. diff --git a/docs/topics/optimize.rst b/docs/topics/optimize.rst new file mode 100644 index 000000000..cea49fca3 --- /dev/null +++ b/docs/topics/optimize.rst @@ -0,0 +1,353 @@ +.. _optimize: + +============ +Optimization +============ + +A crawl goes as fast as its slowest part allows. :ref:`Find out which part that +is ` before changing any setting. + +:ref:`Broad crawls ` have their own set of recommended +adjustments. + +.. _optimize-bottleneck: + +Finding the bottleneck +====================== + +The bottleneck depends on the spider: on the same machine, one crawl can be +limited by its own parsing code and another by the target website. So measure +the crawl that you want to optimize. + +:class:`~scrapy.extensions.logstats.LogStats` reports crawl speed every +:setting:`LOGSTATS_INTERVAL` seconds: + +.. code-block:: text + + [scrapy.extensions.logstats] INFO: Crawled 1200 pages (at 60 pages/min), scraped 1150 items (at 58 items/min) + +A rate that stays flat as you raise :setting:`CONCURRENT_REQUESTS` means +something else is the limit. + + +Reading the engine status +------------------------- + +The :ref:`telnet console ` reports, through ``est()``, +what every part of the engine is doing at a given moment: + +.. code-block:: text + + len(engine.downloader.active) : 16 + len(engine._slot.scheduler.mqs) : 92 + len(engine.scraper.slot.active) : 0 + engine.scraper.slot.active_size : 0 + engine.scraper.slot.needs_backout() : False + +Take a few readings at different points of the crawl: + +- ``len(engine.downloader.active)`` stays at :setting:`CONCURRENT_REQUESTS`: + the downloader is the limit. You are waiting on the network or on the + target website. See :ref:`optimize-concurrency`. + +- ``len(engine.downloader.active)`` stays below + :setting:`CONCURRENT_REQUESTS` while the scheduler queues (``mqs``, + ``dqs``) hold requests: something throttles those requests before they + reach the downloader, usually :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, + :setting:`DOWNLOAD_DELAY` or :ref:`AutoThrottle `. + +- Both the downloader and the scheduler queues stay near empty: your spider + is not producing requests fast enough. A crawl that walks pagination one + page at a time cannot use more concurrency than it creates. See + :ref:`optimize-requests`. + +- ``needs_backout()`` is ``True``, or ``active_size`` approaches + :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE`: responses arrive faster than your + callbacks and :ref:`item pipelines ` handle them. The + bottleneck is your own code. + +- ``len(engine._slot.scheduler.mqs)`` grows without settling: the crawl + discovers requests faster than it downloads them. This is what makes long + crawls run out of memory. + + +Reading resource usage +---------------------- + +CPU + Scrapy runs in a single process, and everything except DNS resolution and + code you explicitly move to a thread runs in a single thread. One CPU core + is the ceiling; a process sitting at 100% of a core is CPU-bound no matter + how many cores the machine has. + + Use a sampling profiler, such as py-spy_, to find out which code is + spending that CPU. :ref:`Selectors ` and item pipelines + are the usual answer. + + .. _py-spy: https://github.com/benfred/py-spy + +Memory + The :ref:`memory usage extension ` records + :stat:`memusage/startup` and :stat:`memusage/max`. A :stat:`memusage/max` + far above :stat:`memusage/startup` is expected; what matters is whether it + keeps growing for as long as the crawl runs. + + Growth that tracks ``len(engine._slot.scheduler.mqs)`` is a scheduling + problem, covered in :ref:`optimize-memory`. Growth that does not is a + :ref:`memory leak `. + +Network + Compare :stat:`downloader/response_bytes` over the crawl time against your + available bandwidth. Saturated bandwidth caps concurrency regardless of any + setting. + + DNS resolution is separate: it runs on a thread pool of + :setting:`REACTOR_THREADPOOL_MAXSIZE` threads, and results are cached + (:setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE`). It only becomes a + limit of its own when there are many different domains to resolve, as in + :ref:`broad crawls `, where it shows up as slow starts and + DNS timeouts. + +Disk + :ref:`Feed exports ` write to disk on most crawls, + although item data is usually small enough for that not to matter. The ones + to suspect are + :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` and + the :ref:`media pipelines `, which write whole + responses, and :setting:`JOBDIR`, which writes every scheduled request. + + +.. _optimize-concurrency: + +Sending more requests at a time +=============================== + +:setting:`CONCURRENT_REQUESTS` caps how many requests are being downloaded at +any given moment, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` caps how many of +those may target the same domain, and :setting:`DOWNLOAD_DELAY` sets a minimum +wait between two consecutive requests to the same domain. A project generated by +:command:`startproject` gets one request per second per domain out of these. + +Raise them to crawl a single website faster, and see +:ref:`broad-crawls-concurrency` to spread requests across many websites +instead. + +The limit that matters, though, is the one the target website tolerates. +Exceeding it gets you throttled, served errors or banned, all of which make the +crawl slower than a lower concurrency would have been. To find that limit: + +- Read the :ref:`robots.txt ` file of the website. Scrapy + does not act on its ``Crawl-delay`` and ``Request-rate`` directives, so when + they are present, translate them into :setting:`DOWNLOAD_DELAY` and + concurrency settings yourself. + +- Check the traffic that the website already gets, using a service like + `SimilarWeb`_ or `Cloudflare Radar`_. A rate that is a rounding error next + to what the website serves anyway is unlikely to be a problem for it. + + .. _SimilarWeb: https://www.similarweb.com/ + .. _Cloudflare Radar: https://radar.cloudflare.com/ + +- Look for a documented way in. An API, a bulk export or a search endpoint is + both faster for you and cheaper for the website than crawling its pages, and + the terms of service may state a rate. + +- Crawl when the website is idle, in its own timezone, so that the capacity + you take is capacity nobody else wanted. + +- Raise concurrency gradually and watch the website respond. + :stat:`downloader/response_status_count/{status_code}` counts for 429, 503 + or the ban page of the website, growing :stat:`retry/count`, or a + :ref:`download latency ` that climbs as you push harder, + all mean you have gone past the limit. + + +.. _optimize-requests: + +Producing requests faster +========================= + +A spider that discovers its requests one response at a time keeps the +downloader idle no matter how high you set :setting:`CONCURRENT_REQUESTS`. To +put more requests in the scheduler earlier: + +- Request every page at once when you can work out how many there are, e.g. + from a page count or from a result count and a page size in the first + response, instead of following a link to the next page on every response. + +- Get URLs from a source that lists many of them at once, such as a sitemap + or a search or export endpoint of the target website. For a crawl that + needs nothing else, :class:`~scrapy.spiders.SitemapSpider` reads sitemaps + for you. + +- Raise the :attr:`~scrapy.Request.priority` of pagination requests, so that + they are downloaded before the requests that they compete with, and + discover the rest of the crawl sooner. + +Each of these trades memory for speed: a request produced before the downloader +can take it waits in the scheduler, or on disk if you set :setting:`JOBDIR`. +Pushed far enough, they turn memory or disk into your new bottleneck, which is +why :ref:`optimize-memory` recommends the reverse of the last point. + + +.. _optimize-resources: + +Lowering resource usage +======================= + +.. _optimize-memory: + +Lowering memory usage +--------------------- + +- Lower :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE`. + +- Lower :setting:`DOWNLOAD_MAXSIZE`, which allows a single response to take up + to 1 GiB of memory by default, multiplied by your concurrency. Set + :setting:`DOWNLOAD_WARNSIZE` first to find out whether the website actually + serves responses that big. + +- Lower the number of :ref:`scheduled requests ` held in + memory: + + - Increase the :attr:`~scrapy.Request.priority` of requests whose + :attr:`~scrapy.Request.callback` cannot yield additional requests. + + For example, the following spider uses a higher priority (1) for book + requests than for pagination requests: + + .. code-block:: python + + from scrapy import Spider + + + class BooksToScrapeComSpider(Spider): + name = "books_toscrape_com" + start_urls = [ + "http://books.toscrape.com/catalogue/category/books/mystery_3/index.html" + ] + + def parse(self, response): + next_page_links = response.css(".next a") + yield from response.follow_all(next_page_links) + book_links = response.css("article a") + yield from response.follow_all(book_links, callback=self.parse_book, priority=1) + + def parse_book(self, response): + yield { + "name": response.css("h1::text").get(), + "price": response.css(".price_color::text").re_first("£(.*)"), + "url": response.url, + } + + .. note:: If the number of request-yielding, low-priority requests + scheduled at any given time is lower than concurrency settings + (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or + :setting:`CONCURRENT_REQUESTS`), as in the example above, this can + slow down your crawl by turning those requests into a bottleneck. + + - If you have many :ref:`start requests `, consider + :ref:`delaying their iteration `. + + - Set :setting:`JOBDIR` to offload all scheduled requests to disk. + +- Be on the lookout for :ref:`memory leaks `. + + +Lowering network usage +---------------------- + +- Install brotli_ and zstandard_ to support brotli-compressed_ and + zstd-compressed_ responses. + + .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt + .. _brotli: https://pypi.org/project/Brotli/ + .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt + .. _zstandard: https://pypi.org/project/zstandard/ + +- Enable :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + while developing your spider, so that re-runs do not download the same + responses again. + + +Lowering CPU usage +------------------ + +- Set :setting:`LOG_LEVEL` to ``"INFO"`` or higher. + +- Restrict what you parse. A :ref:`selector ` over a + smaller part of the response, or a single query whose result you reuse, + beats repeated queries over the whole document. + + +Other tips +---------- + +- Try :ref:`using the asyncio reactor ` with uvloop_ as + :ref:`custom event loop `, i.e. setting + :setting:`ASYNCIO_EVENT_LOOP` to ``"uvloop.Loop"``. + + .. _uvloop: https://github.com/MagicStack/uvloop + + Alternatively, try :ref:`switching to a non-asyncio reactor + `. + +- Disable unused :ref:`components `. + + For example, set :setting:`COOKIES_ENABLED` to ``False`` unless you need + cookies. + +- Split the crawl across separate processes to use more than one CPU core. + See :ref:`distributed-crawls`. + + +.. _broad-crawls: +.. _topics-broad-crawls: + +Speeding up broad crawls +======================== + +While Scrapy is well suited for **broad crawls**, i.e. crawls that target many +websites, the default :ref:`settings ` are optimized for +crawls targeting a single website. + +For broad crawls, consider these adjustments: + +- .. _broad-crawls-concurrency: + + Increase the global concurrency: + + - Set :setting:`CONCURRENT_REQUESTS` as close to + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` × [number of target domains] + (e.g. 8 × 10 domains = 80 concurrent requests) as your CPU and memory + allow. + + - Increase :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` when increasing + :setting:`CONCURRENT_REQUESTS` stops making a difference. + +- .. _broad-crawls-bfo: + + If memory is a bottleneck, see if :ref:`crawling in BFO order ` lowers + memory usage. + +- Improve DNS resolution speed: + + - Set up your own DNS server, with a local cache and upstream to a `large + DNS server`_, to avoid slowing down your network. + + .. _large DNS server: https://en.wikipedia.org/wiki/Public_recursive_name_server#Notable_public_DNS_service_operators + + - Increase :setting:`REACTOR_THREADPOOL_MAXSIZE` to the minimum value + that avoids DNS resolution timeouts and makes a noticeable positive + impact in crawl speed. + +- Lower the negative impact of some responses: + + - Set :setting:`RETRY_ENABLED` to ``False`` or, if you need retries, + consider lowering :setting:`RETRY_TIMES`. + + - Lower :setting:`DOWNLOAD_TIMEOUT` to a more reasonable value, to + discard stuck requests more quickly. + + - Set :setting:`REDIRECT_ENABLED` to ``False`` unless you want to follow + redirects. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ec222d999..0ea60e9cf 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1913,6 +1913,7 @@ Type of in-memory queue used by the scheduler. Other available type is: .. setting:: SCHEDULER_PRIORITY_QUEUE +.. _broad-crawls-scheduler-priority-queue: SCHEDULER_PRIORITY_QUEUE ------------------------ From 609f64c55d8603710218113ad42b1bbf940b3978 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 19:24:14 +0200 Subject: [PATCH 087/111] Make brotli a hard dependency (#7929) --- .github/workflows/tests-ubuntu.yml | 1 + docs/intro/install.rst | 2 - docs/topics/downloader-middleware.rst | 9 ++- pyproject.toml | 6 +- .../downloadermiddlewares/httpcompression.py | 24 +------- scrapy/utils/_compression.py | 9 ++- tests/test_command_shell.py | 6 +- ...st_downloadermiddleware_httpcompression.py | 55 ------------------- tox.ini | 8 +-- 9 files changed, 18 insertions(+), 102 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index cd726a2fe..b58b2af7b 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -86,6 +86,7 @@ jobs: - python-version: pypy3.11-7.3.20 env: TOXENV: pypy3-extra-deps + coverage: true - python-version: "3.14" env: TOXENV: botocore diff --git a/docs/intro/install.rst b/docs/intro/install.rst index cba8c15a1..866c5abe2 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -111,8 +111,6 @@ The following extras are available: - Provides * - ``bpython`` - :ref:`bpython shell ` - * - ``brotli`` - - :ref:`Brotli response decompression ` * - ``gcs`` - :ref:`Google Cloud Storage ` for :ref:`feed exports ` and diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index fcfe7fd29..66a0164d9 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -741,14 +741,13 @@ HttpCompressionMiddleware .. class:: HttpCompressionMiddleware - This middleware allows compressed (gzip, deflate) traffic to be + This middleware allows compressed (gzip, deflate, `brotli`_) traffic to be sent/received from web sites. - This middleware also supports decoding `brotli-compressed`_ responses with - the :ref:`brotli ` extra, and `zstd-compressed`_ - responses with the :ref:`zstd ` extra. + This middleware also supports decoding `zstd-compressed`_ responses with + the :ref:`zstd ` extra. -.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt +.. _brotli: https://www.ietf.org/rfc/rfc7932.txt .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt diff --git a/pyproject.toml b/pyproject.toml index 0dcbade90..4edfb7ec8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,8 @@ dependencies = [ # Platform-specific dependencies 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', + 'brotli>=1.2.0; implementation_name != "pypy"', + 'brotlicffi>=1.2.0.0; implementation_name == "pypy"', ] classifiers = [ "Development Status :: 5 - Production/Stable", @@ -62,10 +64,6 @@ Tracker = "https://github.com/scrapy/scrapy/issues" [project.optional-dependencies] bpython = ["bpython>=0.7.1"] -brotli = [ - "brotli>=1.2.0; implementation_name != 'pypy'", - "brotlicffi>=1.2.0.0; implementation_name == 'pypy'", -] gcs = ["google-cloud-storage>=1.29.0"] httpx = ["httpx2[http2,socks]>=2.0.0"] images = ["Pillow>=8.3.2"] diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index b99c323c5..0045ddcaa 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -30,27 +30,7 @@ if TYPE_CHECKING: logger = getLogger(__name__) -ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate"] - -try: - try: - import brotli - except ImportError: - import brotlicffi as brotli -except ImportError: - pass -else: - try: - brotli.Decompressor.can_accept_more_data # noqa: B018 - except AttributeError: # pragma: no cover - warnings.warn( - "You have brotli installed. But 'br' encoding support now requires " - "brotli's or brotlicffi's version >= 1.2.0. Please upgrade " - "brotli/brotlicffi to make Scrapy decode 'br' encoded responses.", - stacklevel=2, - ) - else: - ACCEPTED_ENCODINGS.append(b"br") +ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate", b"br"] if find_spec("zstandard") is not None: ACCEPTED_ENCODINGS.append(b"zstd") @@ -205,8 +185,6 @@ class HttpCompressionMiddleware: f"{self.__class__.__name__} cannot decode the response for {response.url} " f"from unsupported encoding(s) '{encodings_str}'." ) - if b"br" in encodings: - msg += " You need to install brotli or brotlicffi >= 1.2.0 to decode 'br'." if b"zstd" in encodings: msg += " You need to install zstandard to decode 'zstd'." logger.warning(msg) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 4767c29f2..ac98a61c8 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -2,11 +2,10 @@ import contextlib import zlib from io import BytesIO -with contextlib.suppress(ImportError): - try: - import brotli - except ImportError: - import brotlicffi as brotli +try: + import brotli +except ImportError: + import brotlicffi as brotli with contextlib.suppress(ImportError): import zstandard diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 29667a1ae..6bc1ebbbb 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -201,7 +201,7 @@ class TestInteractiveShell: env = os.environ.copy() env["SCRAPY_PYTHON_SHELL"] = "python" logfile = BytesIO() - p = PopenSpawn(args, env=env, timeout=5) + p = PopenSpawn(args, env=env, timeout=60) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") p.sendline(f"fetch('{mockserver.url('/')}')") @@ -235,7 +235,7 @@ class TestInteractiveShell: def _run_interactive_shell(self, env: dict[str, str]) -> str: args = (sys.executable, "-m", "scrapy.cmdline", "shell") logfile = BytesIO() - p = PopenSpawn(args, env=env, timeout=5) + p = PopenSpawn(args, env=env, timeout=60) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") p.sendeof() @@ -256,7 +256,7 @@ class TestInteractiveShell: self._isolate_config(env, config_home) args = (sys.executable, "-m", "scrapy.cmdline", "shell") logfile = BytesIO() - p = PopenSpawn(args, env=env, timeout=10) + p = PopenSpawn(args, env=env, timeout=60) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") # The standard Python shell never imports IPython, whereas the IPython diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index fa0707491..a43bb51ba 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -52,20 +52,6 @@ FORMAT = { } -def _skip_if_no_br() -> None: - try: - try: - import brotli # noqa: PLC0415 - - brotli.Decompressor.can_accept_more_data - except (ImportError, AttributeError): - import brotlicffi # noqa: PLC0415 - - brotlicffi.Decompressor.can_accept_more_data - except (ImportError, AttributeError): - pytest.skip("no brotli support") - - def _skip_if_no_zstd() -> None: pytest.importorskip("zstandard") @@ -161,8 +147,6 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_bytes", 74837) def test_process_response_br(self): - _skip_if_no_br() - response = self._getresponse("br") assert response.request request = response.request @@ -174,32 +158,6 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 74837) - def test_process_response_br_unsupported(self, caplog: pytest.LogCaptureFixture): - if find_spec("brotli") is not None or find_spec("brotlicffi") is not None: - pytest.skip("Requires not having brotli support") - response = self._getresponse("br") - assert response.request - request = response.request - assert response.headers["Content-Encoding"] == b"br" - caplog.clear() - with caplog.at_level( - WARNING, logger="scrapy.downloadermiddlewares.httpcompression" - ): - newresponse = self.mw.process_response(request, response) - assert caplog.record_tuples == [ - ( - "scrapy.downloadermiddlewares.httpcompression", - WARNING, - ( - "HttpCompressionMiddleware cannot decode the response for " - "http://scrapytest.org/ from unsupported encoding(s) 'br'. " - "You need to install brotli or brotlicffi >= 1.2.0 to decode 'br'." - ), - ), - ] - assert newresponse is not response - assert newresponse.headers.getlist("Content-Encoding") == [b"br"] - def test_process_response_zstd(self): _skip_if_no_zstd() @@ -550,8 +508,6 @@ class TestHttpCompression: assert cause.decompressed_size < 1_100_000 def test_compression_bomb_setting_br(self): - _skip_if_no_br() - self._test_compression_bomb_setting("br") def test_compression_bomb_setting_deflate(self): @@ -609,8 +565,6 @@ class TestHttpCompression: @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_compression_bomb_spider_attr_br(self): - _skip_if_no_br() - self._test_compression_bomb_spider_attr("br") @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") @@ -643,8 +597,6 @@ class TestHttpCompression: assert cause.decompressed_size < 1_100_000 def test_compression_bomb_request_meta_br(self): - _skip_if_no_br() - self._test_compression_bomb_request_meta("br") def test_compression_bomb_request_meta_deflate(self): @@ -689,8 +641,6 @@ class TestHttpCompression: def test_download_warnsize_setting_br( self, caplog: pytest.LogCaptureFixture ) -> None: - _skip_if_no_br() - self._test_download_warnsize_setting(caplog, "br") def test_download_warnsize_setting_deflate( @@ -744,8 +694,6 @@ class TestHttpCompression: def test_download_warnsize_spider_attr_br( self, caplog: pytest.LogCaptureFixture ) -> None: - _skip_if_no_br() - self._test_download_warnsize_spider_attr(caplog, "br") @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") @@ -799,8 +747,6 @@ class TestHttpCompression: def test_download_warnsize_request_meta_br( self, caplog: pytest.LogCaptureFixture ) -> None: - _skip_if_no_br() - self._test_download_warnsize_request_meta(caplog, "br") def test_download_warnsize_request_meta_deflate( @@ -834,7 +780,6 @@ class TestHttpCompression: return new_response def test_process_truncated_response_br(self): - _skip_if_no_br() resp = self._get_truncated_response("br") assert resp.body.startswith(b"= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests - brotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests google-cloud-storage httpx2[http2,socks] ipython @@ -189,8 +189,6 @@ deps = Twisted[http2]==21.7.0 boto3==1.20.0 bpython==0.7.1 - brotli==1.2.0; implementation_name != "pypy" - brotlicffi==1.2.0.0; implementation_name == "pypy" google-cloud-storage==1.29.0 httpx2[http2,socks]==2.0.0 ipython==8.15.0 @@ -291,7 +289,6 @@ commands = basepython = pypy3 deps = {[testenv:extra-deps]deps} -commands = {[testenv:pypy3]commands} [testenv:min-pypy3] basepython = pypy3.11 @@ -301,6 +298,7 @@ deps = pytest==8.4.0 Protego==0.1.15 Twisted==21.7.0 + brotlicffi==1.2.0.0 cryptography==44.0.2 cssselect==0.9.1 itemadapter==0.1.0 From 7d9516f33292c8f81a41c9b113f3a135a66c8384 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 19:30:30 +0200 Subject: [PATCH 088/111] Canonicalize each extracted link once (#7961) --- scrapy/linkextractors/lxmlhtml.py | 2 +- tests/test_linkextractors.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 46ac39a28..75f9753e7 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -332,7 +332,7 @@ class LxmlLinkExtractor: unique=unique, process=process_value, strip=strip, - canonicalized=not canonicalize, + canonicalized=True, ) self.allow_res: list[re.Pattern[str]] = self._compile_regexes(allow) self.deny_res: list[re.Pattern[str]] = self._compile_regexes(deny) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 95a6aee54..7b73a133f 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -9,6 +9,7 @@ from w3lib import __version__ as w3lib_version from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link +from scrapy.linkextractors import lxmlhtml from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor, LxmlParserLinkExtractor from tests import get_testdata @@ -798,6 +799,25 @@ class Base: class TestLxmlLinkExtractor(Base.TestLinkExtractorBase): extractor_cls = LxmlLinkExtractor + def test_canonicalize_once_per_link(self, monkeypatch): + canonicalize_url = lxmlhtml.canonicalize_url + calls = [] + + def counting_canonicalize_url(url, *args, **kwargs): + calls.append(url) + return canonicalize_url(url, *args, **kwargs) + + monkeypatch.setattr(lxmlhtml, "canonicalize_url", counting_canonicalize_url) + response = HtmlResponse( + "https://example.com", + body=b"".join(b'x' % i for i in range(10)), + ) + lx = self.extractor_cls(canonicalize=True) + assert lx.extract_links(response) == [ + Link(url="https://example.com/p?a=1&b=2", text="x") + ] + assert len(calls) == 10 + def test_link_restrict_text(self): html = b""" Pic of a cat From 786ab1049431bd890bbf7b0b35551037b385d04a Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 19:31:53 +0200 Subject: [PATCH 089/111] Document how to write a custom item exporter (#7931) --- docs/topics/exporters.rst | 66 +++++++++++++++++++++++++++++++++++++++ scrapy/exporters.py | 27 +++++++++------- tests/test_exporters.py | 12 +++---- 3 files changed, 88 insertions(+), 17 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index c43b7e20f..56b995e18 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -136,6 +136,70 @@ Example: return f"$ {str(value)}" return super().serialize_field(field, name, value) +.. _custom-exporters: + +Writing your own item exporter +============================== + +To write an item exporter, subclass :class:`BaseItemExporter` and implement +:meth:`~BaseItemExporter.export_item`, where +:meth:`~BaseItemExporter.get_serialized_fields` gives you the ``(name, value)`` +pairs to export. + +To make your exporter available to the :ref:`feed exports +`, list it in the :setting:`FEED_EXPORTERS` setting. Feed +exports :ref:`build ` it with the output file as the first +positional argument, and with the ``fields``, ``encoding`` and ``indent`` +:ref:`feed options ` and every key of ``item_export_kwargs`` as +keyword arguments, so your ``__init__`` method must forward unknown keyword +arguments to :class:`BaseItemExporter`. + +The file object belongs to whoever opened it, i.e. to the feed storage in the +case of feed exports, which also closes it. If you need a text file, for +example to use :func:`csv.writer` or another Python API that does not accept a +binary file, wrap it with :class:`io.TextIOWrapper` and call +:meth:`~io.TextIOBase.detach` on the wrapper in +:meth:`~BaseItemExporter.finish_exporting`; otherwise the wrapper closes the +underlying file when it is garbage-collected. + +For example, the following item exporter writes items as blocks of +``name: value`` lines: + +.. code-block:: python + + from io import TextIOWrapper + + from scrapy.exporters import BaseItemExporter + + + class TextItemExporter(BaseItemExporter): + def __init__(self, file, item_separator="\n", **kwargs): + super().__init__(**kwargs) + self.item_separator = item_separator + self.stream = TextIOWrapper( + file, encoding=self.encoding or "utf-8", write_through=True + ) + + def export_item(self, item): + for name, value in self.get_serialized_fields(item): + print(f"{name}: {value}", file=self.stream) + self.stream.write(self.item_separator) + + def finish_exporting(self): + self.stream.detach() + +To use it as the ``txt`` feed format: + +.. code-block:: python + + FEED_EXPORTERS = {"txt": "myproject.exporters.TextItemExporter"} + FEEDS = { + "items.txt": { + "format": "txt", + "item_export_kwargs": {"item_separator": "---\n"}, + }, + } + .. _topics-exporters-reference: Built-in Item Exporters reference @@ -168,6 +232,8 @@ BaseItemExporter Exports the given item. This method must be implemented in subclasses. + .. automethod:: BaseItemExporter.get_serialized_fields + .. method:: serialize_field(field, name, value) Return the serialized value for the given field. You can override this diff --git a/scrapy/exporters.py b/scrapy/exporters.py index ea600d1a8..7f8aaf059 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -85,11 +85,16 @@ class BaseItemExporter(ABC): declared = (name for name in adapter.field_names() if name in populated) return dict.fromkeys([*declared, *adapter.keys()]) - def _get_serialized_fields( + def get_serialized_fields( self, item: Any, default_value: Any = None, include_empty: bool | None = None ) -> Iterable[tuple[str, Any]]: - """Return the fields to export as an iterable of tuples - (name, serialized_value) + """Return the fields of *item* to export, as an iterable of + ``(name, serialized_value)`` tuples, taking :attr:`fields_to_export` + into account and applying :meth:`serialize_field` to every value. + + Fields missing from *item* are exported with *default_value*. + + *include_empty* overrides :attr:`export_empty_fields`. """ item = ItemAdapter(item) @@ -136,7 +141,7 @@ class JsonLinesItemExporter(BaseItemExporter): self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs) def export_item(self, item: Any) -> None: - itemdict = dict(self._get_serialized_fields(item)) + itemdict = dict(self.get_serialized_fields(item)) data = self.encoder.encode(itemdict) + "\n" self.file.write(to_bytes(data, self.encoding)) @@ -176,7 +181,7 @@ class JsonItemExporter(BaseItemExporter): self.file.write(b"]") def export_item(self, item: Any) -> None: - itemdict = dict(self._get_serialized_fields(item)) + itemdict = dict(self.get_serialized_fields(item)) data = to_bytes(self.encoder.encode(itemdict), self.encoding) self._add_comma_after_first() self.file.write(data) @@ -216,7 +221,7 @@ class XmlItemExporter(BaseItemExporter): self._beautify_indent(depth=1) self.xg.startElement(self.item_element, AttributesImpl({})) self._beautify_newline() - for name, value in self._get_serialized_fields(item, default_value=""): + for name, value in self.get_serialized_fields(item, default_value=""): self._export_xml_field(name, value, depth=2) self._beautify_indent(depth=1) self.xg.endElement(self.item_element) @@ -310,7 +315,7 @@ class CsvItemExporter(BaseItemExporter): f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields", ) self._data_loss_warned = True - fields = self._get_serialized_fields(item, default_value="", include_empty=True) + fields = self.get_serialized_fields(item, default_value="", include_empty=True) values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) @@ -347,7 +352,7 @@ class PickleItemExporter(BaseItemExporter): self.protocol: int = protocol def export_item(self, item: Any) -> None: - d = dict(self._get_serialized_fields(item)) + d = dict(self.get_serialized_fields(item)) pickle.dump(d, self.file, self.protocol) @@ -365,7 +370,7 @@ class MarshalItemExporter(BaseItemExporter): self.file: BytesIO = file def export_item(self, item: Any) -> None: - marshal.dump(dict(self._get_serialized_fields(item)), self.file) + marshal.dump(dict(self.get_serialized_fields(item)), self.file) class PprintItemExporter(BaseItemExporter): @@ -374,7 +379,7 @@ class PprintItemExporter(BaseItemExporter): self.file: BytesIO = file def export_item(self, item: Any) -> None: - itemdict = dict(self._get_serialized_fields(item)) + itemdict = dict(self.get_serialized_fields(item)) self.file.write(to_bytes(pprint.pformat(itemdict) + "\n")) @@ -417,5 +422,5 @@ class PythonItemExporter(BaseItemExporter): yield key, self._serialize_value(value) def export_item(self, item: Any) -> dict[str | bytes, Any]: # type: ignore[override] - result: dict[str | bytes, Any] = dict(self._get_serialized_fields(item)) + result: dict[str | bytes, Any] = dict(self.get_serialized_fields(item)) return result diff --git a/tests/test_exporters.py b/tests/test_exporters.py index b857728ba..99afb8beb 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -108,26 +108,26 @@ class TestBaseItemExporter(ABC): def test_fields_to_export(self): ie = self._get_exporter(fields_to_export=["name"]) - assert list(ie._get_serialized_fields(self.i)) == [("name", "John\xa3")] + assert list(ie.get_serialized_fields(self.i)) == [("name", "John\xa3")] ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1") - _, name = next(iter(ie._get_serialized_fields(self.i))) + _, name = next(iter(ie.get_serialized_fields(self.i))) assert isinstance(name, str) assert name == "John\xa3" ie = self._get_exporter(fields_to_export={"name": "名稱"}) - assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")] + assert list(ie.get_serialized_fields(self.i)) == [("名稱", "John\xa3")] def test_field_order(self): item = self.item_class(age="22", name="John\xa3") ie = self._get_exporter() - assert [name for name, _ in ie._get_serialized_fields(item)] == ["name", "age"] + assert [name for name, _ in ie.get_serialized_fields(item)] == ["name", "age"] def test_field_order_dict_item(self): ie = self._get_exporter() - assert [name for name, _ in ie._get_serialized_fields({"age": "22"})] == ["age"] + assert [name for name, _ in ie.get_serialized_fields({"age": "22"})] == ["age"] assert [ - name for name, _ in ie._get_serialized_fields({"age": "22", "name": "John"}) + name for name, _ in ie.get_serialized_fields({"age": "22", "name": "John"}) ] == ["age", "name"] def test_field_custom_serializer(self): From 482a02d30c298e67fd610e5f27e56cfe11e9b4c5 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 19:41:32 +0200 Subject: [PATCH 090/111] Add a cookies documentation page (#7947) --- docs/faq.rst | 2 +- docs/index.rst | 4 + docs/topics/cookies.rst | 138 ++++++++++++++++++++++++++ docs/topics/downloader-middleware.rst | 98 +----------------- docs/topics/request-response.rst | 60 +---------- docs/topics/settings.rst | 5 +- 6 files changed, 150 insertions(+), 157 deletions(-) create mode 100644 docs/topics/cookies.rst diff --git a/docs/faq.rst b/docs/faq.rst index 80658a5bf..ef3d07a2b 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -292,7 +292,7 @@ Does Scrapy manage cookies automatically? Yes, Scrapy receives and keeps track of cookies sent by servers, and sends them back on subsequent requests, like any regular web browser does. -For more info see :ref:`topics-request-response` and :ref:`cookies-mw`. +For more info see :ref:`cookies`. How can I see the cookies being sent and received from Scrapy? -------------------------------------------------------------- diff --git a/docs/index.rst b/docs/index.rst index d45b2c208..de06e3488 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -78,6 +78,7 @@ Basic concepts topics/item-pipeline topics/feed-exports topics/request-response + topics/cookies topics/link-extractors topics/settings topics/exceptions @@ -109,6 +110,9 @@ Basic concepts :doc:`topics/request-response` Understand the classes used to represent HTTP requests and responses. +:doc:`topics/cookies` + Send and receive cookies. + :doc:`topics/link-extractors` Convenient classes to extract links to follow from pages. diff --git a/docs/topics/cookies.rst b/docs/topics/cookies.rst new file mode 100644 index 000000000..ab5f3dca0 --- /dev/null +++ b/docs/topics/cookies.rst @@ -0,0 +1,138 @@ +.. _cookies: +.. _cookies-mw: + +======= +Cookies +======= + +Scrapy keeps track of the cookies that websites set and sends them back on +later requests to those websites, just like a web browser does. That is the job +of :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`, which is +enabled by default. + + +Setting cookies on a request +============================ + +.. invisible-code-block: python + + from scrapy import Request + +Use the ``cookies`` parameter of :class:`~scrapy.Request` to send cookies of +your own, either as a dict: + +.. code-block:: python + + request = Request( + url="https://example.com", + cookies={"currency": "USD", "country": "UY"}, + ) + +Or as a list of dicts, which also lets you set cookie attributes: + +.. code-block:: python + + request = Request( + url="https://example.com", + cookies=[ + { + "name": "currency", + "value": "USD", + "domain": "example.com", + "path": "/currency", + "secure": True, + }, + ], + ) + +Setting attributes is only useful if the cookies are stored for later requests, +i.e. if :reqmeta:`dont_merge_cookies` is not enabled. + +.. caution:: Cookies set through the ``Cookie`` header are not handled by + :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`, which + drops that header. + +.. caution:: When a cookie name or value is a byte sequence that is not UTF-8 + encoded, the cookie is dropped and a warning is logged. See + :ref:`topics-logging-advanced-customization` to customize the logging + behavior. + + +.. reqmeta:: cookiejar + +Multiple cookie sessions per spider +=================================== + +By default all requests share a single cookie jar (session). To use different +ones, pass an identifier in the :reqmeta:`cookiejar` request meta key: + +.. skip: next +.. code-block:: python + + for i, url in enumerate(urls): + yield Request(url, meta={"cookiejar": i}, callback=self.parse_page) + +The :reqmeta:`cookiejar` meta key is not "sticky", so you need to keep passing +it along on subsequent requests: + +.. code-block:: python + + def parse_page(self, response): + return Request( + "https://example.com/otherpage", + meta={"cookiejar": response.meta["cookiejar"]}, + callback=self.parse_other_page, + ) + + +.. reqmeta:: dont_merge_cookies + +Skipping the cookie jar for a request +===================================== + +Set the :reqmeta:`dont_merge_cookies` request meta key to ``True`` to keep a +request from touching the cookie jar in either direction: no stored cookie is +sent with the request, and no cookie received in the response is stored. The +cookies of the request itself are ignored as well. + + +.. setting:: COOKIES_ENABLED + +COOKIES_ENABLED +=============== + +Default: ``True`` + +Whether to enable :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`. +If disabled, no cookies are sent to web servers. + + +.. setting:: COOKIES_DEBUG + +COOKIES_DEBUG +============= + +Default: ``False`` + +If enabled, Scrapy logs all cookies sent in requests (i.e. the ``Cookie`` +header) and all cookies received in responses (i.e. the ``Set-Cookie`` +header):: + + 2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened + 2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: + Cookie: clientlanguage_nl=en_EN + 2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> + Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/ + Set-Cookie: ip_isocode=US + Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/ + 2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + [...] + + +CookiesMiddleware +================= + +.. module:: scrapy.downloadermiddlewares.cookies + :synopsis: Cookies Downloader Middleware + +.. autoclass:: CookiesMiddleware diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 66a0164d9..2d6ba0cf3 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -169,106 +169,10 @@ middleware, see the :ref:`downloader middleware usage guide For a list of the components enabled by default (and their orders) see the :setting:`DOWNLOADER_MIDDLEWARES_BASE` setting. -.. _cookies-mw: - CookiesMiddleware ----------------- -.. module:: scrapy.downloadermiddlewares.cookies - :synopsis: Cookies Downloader Middleware - -.. class:: CookiesMiddleware - - This middleware enables working with sites that require cookies, such as - those that use sessions. It keeps track of cookies sent by web servers, and - sends them back on subsequent requests (from that spider), just like web - browsers do. - - .. caution:: When non-UTF8 encoded byte sequences are passed to a - :class:`~scrapy.Request`, the ``CookiesMiddleware`` will log - a warning. Refer to :ref:`topics-logging-advanced-customization` - to customize the logging behaviour. - - .. caution:: Cookies set via the ``Cookie`` header are not considered by the - :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known - current limitation that is being worked on. - -The following settings can be used to configure the cookie middleware: - -* :setting:`COOKIES_ENABLED` -* :setting:`COOKIES_DEBUG` - -.. reqmeta:: cookiejar - -Multiple cookie sessions per spider -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -There is support for keeping multiple cookie sessions per spider by using the -:reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar -(session), but you can pass an identifier to use different ones. - -For example: - -.. skip: next -.. code-block:: python - - for i, url in enumerate(urls): - yield scrapy.Request(url, meta={"cookiejar": i}, callback=self.parse_page) - -Keep in mind that the :reqmeta:`cookiejar` meta key is not "sticky". You need to keep -passing it along on subsequent requests. For example: - -.. code-block:: python - - def parse_page(self, response): - # do some processing - return scrapy.Request( - "http://www.example.com/otherpage", - meta={"cookiejar": response.meta["cookiejar"]}, - callback=self.parse_other_page, - ) - -.. setting:: COOKIES_ENABLED - -COOKIES_ENABLED -~~~~~~~~~~~~~~~ - -Default: ``True`` - -Whether to enable the cookies middleware. If disabled, no cookies will be sent -to web servers. - -Notice that despite the value of :setting:`COOKIES_ENABLED` setting if -``Request.``:reqmeta:`meta['dont_merge_cookies'] ` -evaluates to ``True`` the request cookies will **not** be sent to the -web server and received cookies in :class:`~scrapy.http.Response` will -**not** be merged with the existing cookies. - -For more detailed information see the ``cookies`` parameter in -:class:`~scrapy.Request`. - -.. setting:: COOKIES_DEBUG - -COOKIES_DEBUG -~~~~~~~~~~~~~ - -Default: ``False`` - -If enabled, Scrapy will log all cookies sent in requests (i.e. ``Cookie`` -header) and all cookies received in responses (i.e. ``Set-Cookie`` header). - -Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: - - 2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened - 2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: - Cookie: clientlanguage_nl=en_EN - 2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> - Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/ - Set-Cookie: ip_isocode=US - Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/ - 2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) - [...] +See :ref:`cookies`. DefaultHeadersMiddleware diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a177d1ad5..1d97e39b6 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -53,65 +53,13 @@ Request objects ``None`` is passed as value, the HTTP header will not be sent at all. .. caution:: Cookies set via the ``Cookie`` header are not considered by the - :ref:`cookies-mw`. If you need to set cookies for a request, use the - ``cookies`` argument. This is a known current limitation that is being - worked on. + :ref:`cookie middleware `. If you need to set cookies for a + request, use the ``cookies`` argument. :type headers: dict - :param cookies: the request cookies. These can be sent in two forms. - - .. invisible-code-block: python - - from scrapy import Request - - 1. Using a dict: - - .. code-block:: python - - request_with_cookies = Request( - url="http://www.example.com", - cookies={"currency": "USD", "country": "UY"}, - ) - - 2. Using a list of dicts: - - .. code-block:: python - - request_with_cookies = Request( - url="https://www.example.com", - cookies=[ - { - "name": "currency", - "value": "USD", - "domain": "example.com", - "path": "/currency", - "secure": True, - }, - ], - ) - - The latter form allows for customizing the ``domain`` and ``path`` - attributes of the cookie. This is only useful if the cookies are saved - for later requests. - - .. reqmeta:: dont_merge_cookies - - When some site returns cookies (in a response) those are stored in the - cookies for that domain and will be sent again in future requests. - That's the typical behaviour of any regular web browser. - - Note that setting the :reqmeta:`dont_merge_cookies` key to ``True`` in - :attr:`request.meta ` causes custom cookies to be - ignored. - - For more info see :ref:`cookies-mw`. - - .. caution:: Cookies set via the ``Cookie`` header are not considered by the - :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`scrapy.Request.cookies ` parameter. This is a known - current limitation that is being worked on. - + :param cookies: the request cookies, as a dict of cookie names and values + or as a list of dicts with a cookie each. See :ref:`cookies`. :type cookies: dict or list :param encoding: the encoding of this request (defaults to ``'utf-8'``). diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0ea60e9cf..de665f42e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -654,9 +654,8 @@ The default headers used for Scrapy HTTP Requests. They're populated in the :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`. .. caution:: Cookies set via the ``Cookie`` header are not considered by the - :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known - current limitation that is being worked on. + :ref:`cookie middleware `. If you need to set cookies for a + request, use the :class:`Request.cookies ` parameter. .. caution:: A ``Referer`` header defined here only reaches requests for which :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` does not set From a5614544a2ae3e2efe9d96ff32e476b57d9bec8f Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 19:51:20 +0200 Subject: [PATCH 091/111] Type tests involving items and exports (#7867) --- pyproject.toml | 12 --- scrapy/extensions/feedexport.py | 2 +- scrapy/pipelines/images.py | 4 +- tests/mockserver/ftp.py | 8 +- tests/test_exporters.py | 74 +++++++------ tests/test_feedexport.py | 36 ++++--- tests/test_feedexport_postprocess.py | 18 +++- tests/test_feedexport_storages.py | 50 ++++----- tests/test_feedexport_uri_params.py | 33 ++++-- tests/test_item.py | 44 ++++---- tests/test_loader.py | 6 +- tests/test_pipeline_crawl.py | 10 +- tests/test_pipeline_files.py | 37 ++++--- tests/test_pipeline_images.py | 52 +++++---- tests/test_pipeline_media.py | 156 +++++++++++++++++++-------- tests/test_pipelines.py | 12 ++- 16 files changed, 343 insertions(+), 211 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4edfb7ec8..5ec1d05a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,23 +123,11 @@ module = [ "tests.test_downloaderslotssettings", "tests.test_dupefilters", "tests.test_engine_loop", - "tests.test_exporters", "tests.test_extension_statsmailer", "tests.test_extension_throttle", - "tests.test_feedexport", - "tests.test_feedexport_postprocess", - "tests.test_feedexport_storages", - "tests.test_feedexport_uri_params", - "tests.test_item", "tests.test_linkextractors", - "tests.test_loader", "tests.test_logformatter", "tests.test_mail", - "tests.test_pipeline_crawl", - "tests.test_pipeline_files", - "tests.test_pipeline_images", - "tests.test_pipeline_media", - "tests.test_pipelines", "tests.test_pqueues", "tests.test_scheduler_base", "tests.test_settings", diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fc2b2f43f..e4294ac90 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -149,7 +149,7 @@ class BlockingFeedStorage(ABC): return NamedTemporaryFile(prefix="feed-", dir=path) - def store(self, file: IO[bytes]) -> Deferred[None] | None: + def store(self, file: IO[bytes]) -> Deferred[None]: return deferred_from_coro(run_in_thread(self._store_in_thread, file)) @abstractmethod diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 79b6c4f27..5e7a4b409 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -29,7 +29,7 @@ from scrapy.utils.defer import ensure_awaitable from scrapy.utils.python import to_bytes if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Iterator from os import PathLike from PIL import Image @@ -180,7 +180,7 @@ class ImagesPipeline(FilesPipeline): info: MediaPipeline.SpiderInfo, *, item: Any = None, - ) -> Iterable[tuple[str, Image.Image, BytesIO]]: + ) -> Iterator[tuple[str, Image.Image, BytesIO]]: path = self.file_path(request, response=response, info=info, item=item) orig_image = self._Image.open(BytesIO(response.body)) transposed_image = self._ImageOps.exif_transpose(orig_image) diff --git a/tests/mockserver/ftp.py b/tests/mockserver/ftp.py index 1edd64dda..72760b5ac 100644 --- a/tests/mockserver/ftp.py +++ b/tests/mockserver/ftp.py @@ -27,11 +27,12 @@ class MockFTPServer: (anonymous) and a temporary root path that you can read from the :attr:`path` attribute.""" + proc: Popen[str] + port: int + path: Path + def __init__(self) -> None: - self.proc: Popen[str] | None = None self.host: str = "127.0.0.1" - self.port: int | None = None - self.path: Path | None = None def __enter__(self) -> Self: self.path = Path(mkdtemp()) @@ -63,7 +64,6 @@ class MockFTPServer: traceback: TracebackType | None, ) -> None: rmtree(str(self.path)) - assert self.proc is not None self.proc.kill() self.proc.communicate() diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 99afb8beb..956697a04 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -4,6 +4,7 @@ import marshal import pickle import re from abc import ABC, abstractmethod +from collections.abc import Mapping from datetime import datetime from io import BytesIO from typing import Any @@ -63,18 +64,18 @@ class TestBaseItemExporter(ABC): self.ie = self._get_exporter() @abstractmethod - def _get_exporter(self, **kwargs) -> BaseItemExporter: + def _get_exporter(self, **kwargs: Any) -> BaseItemExporter: raise NotImplementedError - def _check_output(self): # noqa: B027 + def _check_output(self) -> None: # noqa: B027 pass - def _assert_expected_item(self, exported_dict): + def _assert_expected_item(self, exported_dict: dict[str, Any]) -> None: for k, v in exported_dict.items(): exported_dict[k] = to_unicode(v) assert self.i == self.item_class(**exported_dict) - def _get_nonstring_types_item(self): + def _get_nonstring_types_item(self) -> dict[str, Any]: return { "boolean": False, "number": 22, @@ -82,7 +83,7 @@ class TestBaseItemExporter(ABC): "float": 3.14, } - def assertItemExportWorks(self, item): + def assertItemExportWorks(self, item: Any) -> None: self.ie.start_exporting() self.ie.export_item(item) self.ie.finish_exporting() @@ -92,7 +93,7 @@ class TestBaseItemExporter(ABC): del self.ie self._check_output() - def test_export_item(self): + def test_export_item(self) -> None: self.assertItemExportWorks(self.i) def test_export_dict_item(self): @@ -142,7 +143,7 @@ class TestBaseItemExporter(ABC): class TestPythonItemExporter(TestBaseItemExporter): - def _get_exporter(self, **kwargs): + def _get_exporter(self, **kwargs: Any) -> BaseItemExporter: return PythonItemExporter(**kwargs) def test_invalid_option(self): @@ -173,6 +174,7 @@ class TestPythonItemExporter(TestBaseItemExporter): "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], "name": "Jesus", } + assert exported is not None assert isinstance(exported["age"][0], dict) assert isinstance(exported["age"][0]["age"][0], dict) @@ -186,6 +188,7 @@ class TestPythonItemExporter(TestBaseItemExporter): "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], "name": "Jesus", } + assert exported is not None assert isinstance(exported["age"][0], dict) assert isinstance(exported["age"][0]["age"][0], dict) @@ -202,10 +205,10 @@ class TestPythonItemExporterDataclass(TestPythonItemExporter): class TestPprintItemExporter(TestBaseItemExporter): - def _get_exporter(self, **kwargs): + def _get_exporter(self, **kwargs: Any) -> BaseItemExporter: return PprintItemExporter(self.output, **kwargs) - def _check_output(self): + def _check_output(self) -> None: self._assert_expected_item(eval(self.output.getvalue())) @@ -215,10 +218,10 @@ class TestPprintItemExporterDataclass(TestPprintItemExporter): class TestPickleItemExporter(TestBaseItemExporter): - def _get_exporter(self, **kwargs): + def _get_exporter(self, **kwargs: Any) -> BaseItemExporter: return PickleItemExporter(self.output, **kwargs) - def _check_output(self): + def _check_output(self) -> None: self._assert_expected_item(pickle.loads(self.output.getvalue())) def test_export_multiple_items(self): @@ -252,10 +255,10 @@ class TestPickleItemExporterDataclass(TestPickleItemExporter): class TestMarshalItemExporter(TestBaseItemExporter): - def _get_exporter(self, **kwargs): + def _get_exporter(self, **kwargs: Any) -> BaseItemExporter: return MarshalItemExporter(self.output, **kwargs) - def _check_output(self): + def _check_output(self) -> None: self.output.seek(0) self._assert_expected_item(marshal.load(self.output)) @@ -279,7 +282,7 @@ class TestMarshalItemExporterDataclass(TestMarshalItemExporter): class TestCsvItemExporter(TestBaseItemExporter): - def _get_exporter(self, **kwargs): + def _get_exporter(self, **kwargs: Any) -> BaseItemExporter: # We need a fresh instance for each exporter, because # CsvItemExporter.stream.__del__() closes the underlying file # (CsvItemExporter.finish_exporting() calls detach() but not all tests @@ -287,8 +290,10 @@ class TestCsvItemExporter(TestBaseItemExporter): self.output = BytesIO() return CsvItemExporter(self.output, **kwargs) - def assertCsvEqual(self, first, second, msg=None): - def split_csv(csv): + def assertCsvEqual( + self, first: bytes | str, second: bytes | str, msg: str | None = None + ) -> None: + def split_csv(csv: bytes | str) -> list[list[str]]: return [ sorted(re.split(r"(,|\s+)", line)) for line in to_unicode(csv).splitlines(True) @@ -296,13 +301,15 @@ class TestCsvItemExporter(TestBaseItemExporter): assert split_csv(first) == split_csv(second), msg - def _check_output(self): + def _check_output(self) -> None: self.output.seek(0) self.assertCsvEqual( to_unicode(self.output.read()), "age,name\r\n22,John\xa3\r\n" ) - def assertExportResult(self, item, expected, **kwargs): + def assertExportResult( + self, item: Any, expected: bytes | str = b"", **kwargs: Any + ) -> None: fp = BytesIO() ie = CsvItemExporter(fp, **kwargs) ie.start_exporting() @@ -383,7 +390,6 @@ class TestCsvItemExporter(TestBaseItemExporter): with pytest.raises(UnicodeEncodeError): self.assertExportResult( item={"text": "W\u0275\u200brd"}, - expected=None, encoding="windows-1251", ) @@ -417,7 +423,7 @@ class TestCsvItemExporterDataclass(TestCsvItemExporter): class TestXmlItemExporter(TestBaseItemExporter): - def _get_exporter(self, **kwargs): + def _get_exporter(self, **kwargs: Any) -> BaseItemExporter: # We need a fresh instance for each exporter, because # XmlItemExporter.stream.__del__() closes the underlying file # (XmlItemExporter.finish_exporting() calls detach() but not all tests @@ -425,20 +431,22 @@ class TestXmlItemExporter(TestBaseItemExporter): self.output = BytesIO() return XmlItemExporter(self.output, **kwargs) - def assertXmlEquivalent(self, first, second, msg=None): - def xmltuple(elem): + def assertXmlEquivalent( + self, first: bytes, second: bytes, msg: str | None = None + ) -> None: + def xmltuple(elem: Any) -> list[Any]: children = list(elem.iterchildren()) if children: return [(child.tag, sorted(xmltuple(child))) for child in children] return [(elem.tag, [(elem.text, ())])] - def xmlsplit(xmlcontent): + def xmlsplit(xmlcontent: bytes) -> list[Any]: doc = lxml.etree.fromstring(xmlcontent) return xmltuple(doc) assert xmlsplit(first) == xmlsplit(second), msg - def assertExportResult(self, item, expected_value): + def assertExportResult(self, item: Any, expected_value: bytes) -> None: fp = BytesIO() ie = XmlItemExporter(fp) ie.start_exporting() @@ -447,7 +455,7 @@ class TestXmlItemExporter(TestBaseItemExporter): del ie # See the first “del self.ie” in this file for context. self.assertXmlEquivalent(fp.getvalue(), expected_value) - def _check_output(self): + def _check_output(self) -> None: expected_value = ( b'\n' b"22John\xc2\xa3" @@ -538,10 +546,10 @@ class TestJsonLinesItemExporter(TestBaseItemExporter): "age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}}, } - def _get_exporter(self, **kwargs): + def _get_exporter(self, **kwargs: Any) -> BaseItemExporter: return JsonLinesItemExporter(self.output, **kwargs) - def _check_output(self): + def _check_output(self) -> None: exported = json.loads(to_unicode(self.output.getvalue().strip())) assert exported == ItemAdapter(self.i).asdict() @@ -582,14 +590,14 @@ class TestJsonLinesItemExporterDataclass(TestJsonLinesItemExporter): class TestJsonItemExporter(TestJsonLinesItemExporter): _expected_nested = [TestJsonLinesItemExporter._expected_nested] - def _get_exporter(self, **kwargs): + def _get_exporter(self, **kwargs: Any) -> BaseItemExporter: return JsonItemExporter(self.output, **kwargs) - def _check_output(self): + def _check_output(self) -> None: exported = json.loads(to_unicode(self.output.getvalue().strip())) assert exported == [ItemAdapter(self.i).asdict()] - def assertTwoItemsExported(self, item): + def assertTwoItemsExported(self, item: Any) -> None: self.ie.start_exporting() self.ie.export_item(item) self.ie.export_item(item) @@ -658,7 +666,7 @@ class TestJsonItemExporter(TestJsonLinesItemExporter): class TestJsonItemExporterToBytes(TestBaseItemExporter): - def _get_exporter(self, **kwargs): + def _get_exporter(self, **kwargs: Any) -> BaseItemExporter: kwargs["encoding"] = "latin" return JsonItemExporter(self.output, **kwargs) @@ -690,7 +698,9 @@ class TestCustomExporterItem: def test_exporter_custom_serializer(self): class CustomItemExporter(BaseItemExporter): - def serialize_field(self, field, name, value): + def serialize_field( + self, field: Mapping[str, Any] | Field, name: str, value: Any + ) -> Any: if name == "age": return str(int(value) + 1) return super().serialize_field(field, name, value) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 40a763efd..d0f6a297c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -100,6 +100,8 @@ class InstrumentedFeedSlot(FeedSlot): """Instrumented FeedSlot subclass for keeping track of calls to start_exporting and finish_exporting.""" + update_listener: Callable[[str], None] + def start_exporting(self): self.update_listener("start") super().start_exporting() @@ -109,7 +111,7 @@ class InstrumentedFeedSlot(FeedSlot): super().finish_exporting() @classmethod - def subscribe__listener(cls, listener): + def subscribe__listener(cls, listener: IsExportingListener) -> None: cls.update_listener = listener.update @@ -119,7 +121,7 @@ class IsExportingListener: finish_exporting and when a call to finish_exporting has been made before a call to start_exporting.""" - def __init__(self): + def __init__(self) -> None: self.start_without_finish = False self.finish_without_start = False @@ -307,6 +309,7 @@ class TestFeedExport(TestFeedExportBase): } crawler = get_crawler(ItemSpider, settings) yield crawler.crawl(mockserver=self.mockserver) + assert crawler.stats is not None assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 @@ -330,6 +333,7 @@ class TestFeedExport(TestFeedExportBase): side_effect=store, ): yield crawler.crawl(mockserver=self.mockserver) + assert crawler.stats is not None assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1 @@ -347,6 +351,7 @@ class TestFeedExport(TestFeedExportBase): } crawler = get_crawler(ItemSpider, settings) yield crawler.crawl(mockserver=self.mockserver) + assert crawler.stats is not None assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 @@ -487,7 +492,7 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_start_finish_exporting_no_items(self): - items = [] + items: list[Any] = [] settings = { "FEEDS": { self._random_temp_filename(): {"format": "json"}, @@ -526,7 +531,7 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_start_finish_exporting_no_items_exception(self): - items = [] + items: list[Any] = [] settings = { "FEEDS": { self._random_temp_filename(): {"format": "json"}, @@ -611,7 +616,7 @@ class TestFeedExport(TestFeedExportBase): items = [{"foo": "bar"}] header = ["foo"] rows = [{"foo": "bar"}] - settings = {"FEED_EXPORT_FIELDS": []} + settings: dict[str, Any] = {"FEED_EXPORT_FIELDS": []} await self.assertExportedCsv(items, header, rows) await self.assertExportedJsonLines(items, rows, settings) @@ -727,14 +732,14 @@ class TestFeedExport(TestFeedExportBase): def accepts(self, item): return isinstance(item, MyItem) - class CustomFilter2(scrapy.extensions.feedexport.ItemFilter): + class CustomFilter2(ItemFilter): def accepts(self, item): return "foo" in item.fields - class CustomFilter3(scrapy.extensions.feedexport.ItemFilter): + class CustomFilter3(ItemFilter): def accepts(self, item): return ( - isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1" + isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1" # type: ignore[index] ) formats = { @@ -834,7 +839,7 @@ class TestFeedExport(TestFeedExportBase): } for fmt, expected in formats.items(): - settings = { + settings: dict[str, Any] = { "FEEDS": { self._random_temp_filename(): {"format": fmt}, }, @@ -911,7 +916,7 @@ class TestFeedExport(TestFeedExportBase): {"key": "value"}, ] - test_cases = [ + test_cases: list[dict[str, Any]] = [ # JSON { "format": "json", @@ -1132,7 +1137,7 @@ class TestFeedExport(TestFeedExportBase): expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n" expected_without_title_csv = b"FOO,BAR\r\n" - test_cases = [ + test_cases: list[dict[str, Any]] = [ # with title { "options": { @@ -1166,6 +1171,9 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_storage_file_no_postprocessing(self): class Storage: + open_file: IO[bytes] + store_file: IO[bytes] + def __init__(self, uri, *, feed_options=None): pass @@ -1187,6 +1195,10 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_storage_file_postprocessing(self): class Storage: + open_file: IO[bytes] + store_file: IO[bytes] + file_was_closed: bool + def __init__(self, uri, *, feed_options=None): pass @@ -1299,7 +1311,7 @@ class TestItemFilter: class TestFeedExportInit: def test_unsupported_storage(self): - settings = { + settings: dict[str, Any] = { "FEEDS": { "unsupported://uri": {}, }, diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py index 36d8586ce..69696e65e 100644 --- a/tests/test_feedexport_postprocess.py +++ b/tests/test_feedexport_postprocess.py @@ -74,7 +74,13 @@ class TestFeedPostProcessedExports(TestFeedExportBase): return content - def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=""): + def get_gzip_compressed( + self, + data: bytes, + compresslevel: int = 9, + mtime: int = 0, + filename: str = "", + ) -> bytes: data_stream = BytesIO() gzipf = gzip.GzipFile( fileobj=data_stream, @@ -539,11 +545,13 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data = await self.exported_data(self.items, settings) - for filename, result in data.items(): + for filename, data_bytes in data.items(): + expected: Any + result: Any if "pickle" in filename: - expected, result = self.items[0], pickle.loads(result) + expected, result = self.items[0], pickle.loads(data_bytes) elif "marshal" in filename: - expected, result = self.items[0], marshal.loads(result) + expected, result = self.items[0], marshal.loads(data_bytes) else: - expected = filename_to_expected[filename] + expected, result = filename_to_expected[filename], data_bytes assert result == expected diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index 6f9e33449..d7fb5c9c2 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -95,27 +95,34 @@ class TestFileFeedStorage: assert storage.path == path +def get_test_spider(settings: dict[str, Any] | None = None) -> scrapy.Spider: + class TestSpider(scrapy.Spider): + name = "test_spider" + + crawler = get_crawler(settings_dict=settings) + return TestSpider.from_crawler(crawler) + + class TestFTPFeedStorage: - def get_test_spider(self, settings=None): - class TestSpider(scrapy.Spider): - name = "test_spider" - - crawler = get_crawler(settings_dict=settings) - return TestSpider.from_crawler(crawler) - - async def _store(self, uri, content, feed_options=None, settings=None): + async def _store( + self, + uri: str, + content: bytes, + feed_options: dict[str, Any] | None = None, + settings: dict[str, Any] | None = None, + ) -> None: crawler = get_crawler(settings_dict=settings or {}) storage = FTPFeedStorage.from_crawler( crawler, uri, feed_options=feed_options, ) - spider = self.get_test_spider() + spider = get_test_spider() file = storage.open(spider) file.write(content) await maybe_deferred_to_future(storage.store(file)) - def _assert_stored(self, path: Path, content): + def _assert_stored(self, path: Path, content: bytes) -> None: assert path.exists() try: assert path.read_bytes() == content @@ -165,7 +172,7 @@ class TestFTPFeedStorage: def test_uri_auth_quote(self): # RFC3986: 3.2.1. User Information pw_quoted = quote(string.punctuation, safe="") - st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) + st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path") assert st.password == string.punctuation def test_uri_without_hostname(self): @@ -181,24 +188,17 @@ class MyBlockingFeedStorage(BlockingFeedStorage): class TestBlockingFeedStorage: - def get_test_spider(self, settings=None): - class TestSpider(scrapy.Spider): - name = "test_spider" - - crawler = get_crawler(settings_dict=settings) - return TestSpider.from_crawler(crawler) - def test_default_temp_dir(self): b = MyBlockingFeedStorage() - storage_file = b.open(self.get_test_spider()) + storage_file = b.open(get_test_spider()) storage_dir = Path(storage_file.name).parent assert str(storage_dir) == tempfile.gettempdir() def test_temp_file(self, tmp_path): b = MyBlockingFeedStorage() - spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)}) + spider = get_test_spider({"FEED_TEMPDIR": str(tmp_path)}) storage_file = b.open(spider) storage_dir = Path(storage_file.name).parent assert storage_dir == tmp_path @@ -207,7 +207,7 @@ class TestBlockingFeedStorage: b = MyBlockingFeedStorage() invalid_path = tmp_path / "invalid_path" - spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) + spider = get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) with pytest.raises(OSError, match="Not a Directory:"): b.open(spider=spider) @@ -311,7 +311,7 @@ class TestS3FeedStorage: assert storage.access_key == "access_key" assert storage.secret_key == "secret_key" assert storage.region_name == region_name - assert storage.s3_client._client_config.region_name == region_name + assert storage.s3_client._client_config.region_name == region_name # type: ignore[attr-defined] def test_from_crawler_without_acl(self): settings = { @@ -353,7 +353,7 @@ class TestS3FeedStorage: ) assert storage.access_key == "access_key" assert storage.secret_key == "secret_key" - assert storage.s3_client._client_config.region_name == "us-east-1" + assert storage.s3_client._client_config.region_name == "us-east-1" # type: ignore[attr-defined] def test_from_crawler_with_acl(self): settings = { @@ -394,7 +394,7 @@ class TestS3FeedStorage: assert storage.access_key == "access_key" assert storage.secret_key == "secret_key" assert storage.region_name == region_name - assert storage.s3_client._client_config.region_name == region_name + assert storage.s3_client._client_config.region_name == region_name # type: ignore[attr-defined] def test_init_without_max_pool_connections(self) -> None: storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") @@ -497,7 +497,7 @@ class TestGCSFeedStorage: def test_parse_empty_acl(self): pytest.importorskip("google.cloud.storage") - settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""} + settings: dict[str, Any] = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""} crawler = get_crawler(settings_dict=settings) storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") assert storage.acl is None diff --git a/tests/test_feedexport_uri_params.py b/tests/test_feedexport_uri_params.py index 150d8449f..21c291732 100644 --- a/tests/test_feedexport_uri_params.py +++ b/tests/test_feedexport_uri_params.py @@ -2,6 +2,7 @@ from __future__ import annotations import warnings from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any import pytest @@ -10,16 +11,27 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.feedexport import FeedExporter from scrapy.utils.test import get_crawler +if TYPE_CHECKING: + from collections.abc import Callable + + from scrapy.crawler import Crawler + class TestURIParams(ABC): spider_name = "uri_params_spider" deprecated_options = False @abstractmethod - def build_settings(self, uri="file:///tmp/foobar", uri_params=None): + def build_settings( + self, + uri: str = "file:///tmp/foobar", + uri_params: Callable[..., dict[str, Any] | None] | None = None, + ) -> dict[str, Any]: raise NotImplementedError - def _crawler_feed_exporter(self, settings): + def _crawler_feed_exporter( + self, settings: dict[str, Any] + ) -> tuple[Crawler, FeedExporter]: if self.deprecated_options: with pytest.warns( ScrapyDeprecationWarning, @@ -29,6 +41,7 @@ class TestURIParams(ABC): else: crawler = get_crawler(settings_dict=settings) feed_exporter = crawler.get_extension(FeedExporter) + assert feed_exporter is not None return crawler, feed_exporter def test_default(self): @@ -116,8 +129,12 @@ class TestURIParams(ABC): class TestURIParamsSetting(TestURIParams): deprecated_options = True - def build_settings(self, uri="file:///tmp/foobar", uri_params=None): - extra_settings = {} + def build_settings( + self, + uri: str = "file:///tmp/foobar", + uri_params: Callable[..., dict[str, Any] | None] | None = None, + ) -> dict[str, Any]: + extra_settings: dict[str, Any] = {} if uri_params: extra_settings["FEED_URI_PARAMS"] = uri_params return { @@ -129,8 +146,12 @@ class TestURIParamsSetting(TestURIParams): class TestURIParamsFeedOption(TestURIParams): deprecated_options = False - def build_settings(self, uri="file:///tmp/foobar", uri_params=None): - options = { + def build_settings( + self, + uri: str = "file:///tmp/foobar", + uri_params: Callable[..., dict[str, Any] | None] | None = None, + ) -> dict[str, Any]: + options: dict[str, Any] = { "format": "jl", } if uri_params: diff --git a/tests/test_item.py b/tests/test_item.py index 7b4c2e918..45732f157 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,4 +1,5 @@ from abc import ABCMeta +from typing import Any from unittest import mock import pytest @@ -7,9 +8,6 @@ from scrapy.item import Field, Item, ItemMeta class TestItem: - def assertSortedEqual(self, first, second, msg=None): - assert sorted(first) == sorted(second), msg - def test_simple(self): class TestItem(Item): name = Field() @@ -98,16 +96,16 @@ class TestItem: i = TestItem() with pytest.raises(AttributeError): - i.name = "john" + i.name = "john" # type: ignore[assignment] def test_custom_methods(self): class TestItem(Item): name = Field() - def get_name(self): + def get_name(self) -> Any: return self["name"] - def change_name(self, name): + def change_name(self, name: str) -> None: self["name"] = name i = TestItem() @@ -121,40 +119,40 @@ class TestItem: def test_metaclass(self): class TestItem(Item): name = Field() - keys = Field() - values = Field() + keys = Field() # type: ignore[assignment] + values = Field() # type: ignore[assignment] i = TestItem() i["name"] = "John" - assert list(i.keys()) == ["name"] - assert list(i.values()) == ["John"] + assert list(i.keys()) == ["name"] # type: ignore[operator] + assert list(i.values()) == ["John"] # type: ignore[operator] i["keys"] = "Keys" i["values"] = "Values" - self.assertSortedEqual(list(i.keys()), ["keys", "values", "name"]) - self.assertSortedEqual(list(i.values()), ["Keys", "Values", "John"]) + assert sorted(i.keys()) == ["keys", "name", "values"] # type: ignore[operator] + assert sorted(i.values()) == ["John", "Keys", "Values"] # type: ignore[operator] def test_metaclass_with_fields_attribute(self): class TestItem(Item): fields = {"new": Field(default="X")} item = TestItem(new="New") - self.assertSortedEqual(list(item.keys()), ["new"]) - self.assertSortedEqual(list(item.values()), ["New"]) + assert list(item.keys()) == ["new"] + assert list(item.values()) == ["New"] def test_fields_order(self): class TestItem(Item): name = Field() - keys = Field() - values = Field() + keys = Field() # type: ignore[assignment] + values = Field() # type: ignore[assignment] assert list(TestItem.fields) == ["name", "keys", "values"] def test_fields_order_inheritance(self): class ParentItem(Item): name = Field() - keys = Field() - values = Field() + keys = Field() # type: ignore[assignment] + values = Field() # type: ignore[assignment] class TestItem(ParentItem): extra = Field() @@ -169,16 +167,16 @@ class TestItem: def test_metaclass_inheritance(self): class ParentItem(Item): name = Field() - keys = Field() - values = Field() + keys = Field() # type: ignore[assignment] + values = Field() # type: ignore[assignment] class TestItem(ParentItem): keys = Field() i = TestItem() i["keys"] = 3 - assert list(i.keys()) == ["keys"] - assert list(i.values()) == [3] + assert list(i.keys()) == ["keys"] # type: ignore[operator] + assert list(i.values()) == [3] # type: ignore[operator] def test_metaclass_multiple_inheritance_simple(self): class A(Item): @@ -314,7 +312,7 @@ class TestItemMeta: def f(self): # For rationale of this see: # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 - return __class__ + return __class__ # type: ignore[name-defined] MyItem() diff --git a/tests/test_loader.py b/tests/test_loader.py index c094d25d8..969bf5b98 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -79,7 +79,7 @@ class TestBasicItemLoader: class InitializationTestMixin: - item_class: type | None = None + item_class: type def test_keep_single_value(self): """Loaded item should contain values from the initial item""" @@ -311,7 +311,7 @@ class TestSelectortemLoader: def test_init_method_with_base_response(self): """Selector should be None after initialization""" response = Response("https://scrapy.org") - l = ProcessorItemLoader(response=response) + l = ProcessorItemLoader(response=response) # type: ignore[arg-type] assert l.selector is None def test_init_method_with_response(self): @@ -461,6 +461,7 @@ class TestSubselectorLoader: l = NestedItemLoader(response=self.response) nl = l.nested_xpath("//header") + assert nl.selector is not None nl.add_xpath("name", "div/text()") nl.add_css("name_div", "#id") nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall()) @@ -476,6 +477,7 @@ class TestSubselectorLoader: def test_nested_css(self): l = NestedItemLoader(response=self.response) nl = l.nested_css("header") + assert nl.selector is not None nl.add_xpath("name", "div/text()") nl.add_css("name_div", "#id") nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall()) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 0681371ef..8b522255a 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -23,7 +23,10 @@ if TYPE_CHECKING: class MediaDownloadSpider(SimpleSpider): name = "mediadownload" - def _process_url(self, url): + media_key: str + media_urls_key: str + + def _process_url(self, url: str) -> str: return url def parse(self, response): @@ -44,14 +47,15 @@ class MediaDownloadSpider(SimpleSpider): class BrokenLinksMediaDownloadSpider(MediaDownloadSpider): name = "brokenmedia" - def _process_url(self, url): + def _process_url(self, url: str) -> str: return url + ".foo" class RedirectedMediaDownloadSpider(MediaDownloadSpider): name = "redirectedmedia" - def _process_url(self, url): + def _process_url(self, url: str) -> str: + assert self.mockserver return add_or_replace_parameter( self.mockserver.url("/redirect-to"), "goto", url ) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4e7fb118b..3a8a3d0f7 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -22,6 +22,7 @@ from itemadapter import ItemAdapter from twisted.internet.defer import Deferred from twisted.python.failure import Failure +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.item import Field, Item @@ -75,7 +76,7 @@ class DeferredFSFilesStore(FSFilesStore): """A simple store with persist_file() returning a deferred.""" def persist_file(self, path, buf, info, meta=None, headers=None): - deferred = Deferred() + deferred: Deferred[None] = Deferred() # short-hand super() doesn't work in nested functions parent_persist_file = super().persist_file @@ -152,7 +153,7 @@ class TestFilesPipeline: file_path( Request("http://www.dorma.co.uk/images/product_details/2532"), response=Response("http://www.dorma.co.uk/images/product_details/2532"), - info=object(), + info=object(), # type: ignore[arg-type] ) == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1" ) @@ -383,7 +384,7 @@ class TestFilesPipeline: """ class CustomFilesPipeline(FilesPipeline): - def file_path(self, request, response=None, info=None, item=None): + def file_path(self, request, response=None, info=None, item=None) -> str: return f"full/{item.get('path')}" file_path = CustomFilesPipeline.from_crawler( @@ -476,7 +477,7 @@ class TestFilesPipeline: item["file_urls"] = bad_type with pytest.raises(TypeError, match="file_urls must be a list of URLs"): - list(pipeline.get_media_requests(item, None)) + list(pipeline.get_media_requests(item, None)) # type: ignore[arg-type] class TestFilesPipelineFieldsMixin(ABC): @@ -491,10 +492,10 @@ class TestFilesPipelineFieldsMixin(ABC): pipeline = FilesPipeline.from_crawler( get_crawler(None, {"FILES_STORE": tmp_path}) ) - requests = list(pipeline.get_media_requests(item, None)) + requests = list(pipeline.get_media_requests(item, None)) # type: ignore[arg-type] assert requests[0].url == url results = [(True, {"url": url})] - item = pipeline.item_completed(results, item, None) + item = pipeline.item_completed(results, item, None) # type: ignore[arg-type] files = ItemAdapter(item).get("files") assert files == [results[0][1]] assert isinstance(item, self.item_class) @@ -512,10 +513,10 @@ class TestFilesPipelineFieldsMixin(ABC): }, ) ) - requests = list(pipeline.get_media_requests(item, None)) + requests = list(pipeline.get_media_requests(item, None)) # type: ignore[arg-type] assert requests[0].url == url results = [(True, {"url": url})] - item = pipeline.item_completed(results, item, None) + item = pipeline.item_completed(results, item, None) # type: ignore[arg-type] custom_files = ItemAdapter(item).get("custom_files") assert custom_files == [results[0][1]] assert isinstance(item, self.item_class) @@ -581,8 +582,10 @@ class TestFilesPipelineCustomSettings: ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"), } - def _generate_fake_settings(self, tmp_path, prefix=None): - def random_string(): + def _generate_fake_settings( + self, tmp_path: Path, prefix: str | None = None + ) -> dict[str, Any]: + def random_string() -> str: return "".join([chr(random.randint(97, 123)) for _ in range(10)]) settings = { @@ -599,7 +602,7 @@ class TestFilesPipelineCustomSettings: for k, v in settings.items() } - def _generate_fake_pipeline(self): + def _generate_fake_pipeline(self) -> type[FilesPipeline]: class UserDefinedFilePipeline(FilesPipeline): EXPIRES = 1001 FILES_URLS_FIELD = "alfa" @@ -739,14 +742,14 @@ class TestFilesPipelineCustomSettings: def test_file_pipeline_using_pathlike_objects(self, tmp_path): class CustomFilesPipelineWithPathLikeDir(FilesPipeline): - def file_path(self, request, response=None, info=None, *, item=None): - return Path("subdir") / Path(request.url).name + def file_path(self, request, response=None, info=None, *, item=None) -> str: + return str(Path("subdir") / Path(request.url).name) pipeline = CustomFilesPipelineWithPathLikeDir.from_crawler( get_crawler(None, {"FILES_STORE": tmp_path}) ) request = Request("http://example.com/image01.jpg") - assert pipeline.file_path(request) == Path("subdir/image01.jpg") + assert pipeline.file_path(request) == str(Path("subdir/image01.jpg")) class TestFSFilesStore: @@ -1092,7 +1095,7 @@ class TestFTPFileStore: store.port, store.username, store.password, - store.USE_ACTIVE_MODE, + bool(store.USE_ACTIVE_MODE), ) assert data == content @@ -1160,7 +1163,7 @@ class TestBuildFromCrawler: _from_crawler_called = False @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> "Pipeline": settings = crawler.settings store_uri = settings["FILES_STORE"] o = cls(store_uri, crawler=crawler) @@ -1179,7 +1182,7 @@ def test_files_pipeline_raises_notconfigured_when_files_store_invalid(store): settings = Settings() settings.clear() settings.set("FILES_STORE", store, priority="cmdline") - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(settings_dict=dict(settings)) with pytest.raises(NotConfigured): FilesPipeline.from_crawler(crawler) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 19e61579f..43316f125 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -91,7 +91,7 @@ class TestImagesPipeline: file_path( Request("http://www.dorma.co.uk/images/product_details/2532"), response=Response("http://www.dorma.co.uk/images/product_details/2532"), - info=object(), + info=DUMMY_SPIDER_INFO, ) == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg" ) @@ -120,7 +120,7 @@ class TestImagesPipeline: Request("file:///tmp/some.name/foo"), name, response=Response("file:///tmp/some.name/foo"), - info=object(), + info=DUMMY_SPIDER_INFO, ) == "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg" ) @@ -133,7 +133,7 @@ class TestImagesPipeline: class CustomImagesPipeline(ImagesPipeline): def thumb_path( self, request, thumb_id, response=None, info=None, item=None - ): + ) -> str: return f"thumb/{thumb_id}/{item.get('path')}" thumb_path = CustomImagesPipeline.from_crawler( @@ -159,11 +159,23 @@ class TestImagesPipeline: req = Request(url="https://dev.mydeco.com/mydeco.gif") with pytest.raises(ImageException): - next(self.pipeline.get_images(response=resp1, request=req, info=object())) + next( + self.pipeline.get_images( + response=resp1, request=req, info=DUMMY_SPIDER_INFO + ) + ) with pytest.raises(ImageException): - next(self.pipeline.get_images(response=resp2, request=req, info=object())) + next( + self.pipeline.get_images( + response=resp2, request=req, info=DUMMY_SPIDER_INFO + ) + ) with pytest.raises(ImageException): - next(self.pipeline.get_images(response=resp3, request=req, info=object())) + next( + self.pipeline.get_images( + response=resp3, request=req, info=DUMMY_SPIDER_INFO + ) + ) def test_get_images(self): self.pipeline.min_width = 0 @@ -176,7 +188,7 @@ class TestImagesPipeline: req = Request(url="https://dev.mydeco.com/mydeco.gif") get_images_gen = self.pipeline.get_images( - response=resp, request=req, info=object() + response=resp, request=req, info=DUMMY_SPIDER_INFO ) path, new_im, new_buf = next(get_images_gen) @@ -201,7 +213,7 @@ class TestImagesPipeline: req = Request(url="https://dev.mydeco.com/mydeco.gif") get_images_gen = self.pipeline.get_images( - response=resp, request=req, info=object() + response=resp, request=req, info=DUMMY_SPIDER_INFO ) path, new_im, _ = next(get_images_gen) @@ -230,7 +242,7 @@ class TestImagesPipeline: def test_convert_image(self): SIZE = (100, 100) # straight forward case: RGB and JPEG - COLOUR = (0, 127, 255) + COLOUR: tuple[int, ...] = (0, 127, 255) im, buf = _create_image("JPEG", "RGB", SIZE, COLOUR) converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) assert converted.mode == "RGB" @@ -296,7 +308,7 @@ class TestImagesPipeline: item["image_urls"] = bad_type with pytest.raises(TypeError, match="image_urls must be a list of URLs"): - list(pipeline.get_media_requests(item, None)) + list(pipeline.get_media_requests(item, DUMMY_SPIDER_INFO)) class TestImagesPipelineFieldsMixin(ABC): @@ -311,10 +323,10 @@ class TestImagesPipelineFieldsMixin(ABC): pipeline = ImagesPipeline.from_crawler( get_crawler(None, {"IMAGES_STORE": "s3://example/images/"}) ) - requests = list(pipeline.get_media_requests(item, None)) + requests = list(pipeline.get_media_requests(item, DUMMY_SPIDER_INFO)) assert requests[0].url == url - results = [(True, {"url": url})] - item = pipeline.item_completed(results, item, None) + results: Any = [(True, {"url": url})] + item = pipeline.item_completed(results, item, DUMMY_SPIDER_INFO) images = ItemAdapter(item).get("images") assert images == [results[0][1]] assert isinstance(item, self.item_class) @@ -332,10 +344,10 @@ class TestImagesPipelineFieldsMixin(ABC): }, ) ) - requests = list(pipeline.get_media_requests(item, None)) + requests = list(pipeline.get_media_requests(item, DUMMY_SPIDER_INFO)) assert requests[0].url == url - results = [(True, {"url": url})] - item = pipeline.item_completed(results, item, None) + results: Any = [(True, {"url": url})] + item = pipeline.item_completed(results, item, DUMMY_SPIDER_INFO) custom_images = ItemAdapter(item).get("custom_images") assert custom_images == [results[0][1]] assert isinstance(item, self.item_class) @@ -410,13 +422,15 @@ class TestImagesPipelineCustomSettings: "IMAGES_RESULT_FIELD": "images", } - def _generate_fake_settings(self, tmp_path, prefix=None): + def _generate_fake_settings( + self, tmp_path: Path, prefix: str | None = None + ) -> dict[str, Any]: """ :param prefix: string for setting keys :return: dictionary of image pipeline settings """ - def random_string(): + def random_string() -> str: return "".join([chr(random.randint(97, 123)) for _ in range(10)]) settings = { @@ -439,7 +453,7 @@ class TestImagesPipelineCustomSettings: for k, v in settings.items() } - def _generate_fake_pipeline_subclass(self): + def _generate_fake_pipeline_subclass(self) -> type[ImagesPipeline]: """ :return: ImagePipeline class will all uppercase attributes set. """ diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index ba1c18006..50787d7b7 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from typing import TYPE_CHECKING, Any, cast from unittest.mock import MagicMock import pytest @@ -10,7 +11,12 @@ from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.pipelines.files import FileException -from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered +from scrapy.pipelines.media import ( + FileInfo, + FileInfoOrError, + MediaPipeline, + _MediaRequestFiltered, +) from scrapy.utils.defer import _defer_sleep_async from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all @@ -19,21 +25,48 @@ from scrapy.utils.test import get_crawler from tests.utils.decorators import coroutine_test from tests.utils.media_pipelines import mocked_download_func +if TYPE_CHECKING: + from collections.abc import Awaitable + + from twisted.internet.defer import Deferred + + from scrapy.crawler import Crawler + class UserDefinedPipeline(MediaPipeline): - def media_to_download(self, request, info, *, item=None): - pass + def media_to_download( + self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None + ) -> Deferred[FileInfo | None] | None: + return None - def get_media_requests(self, item, info): - pass + def get_media_requests( + self, item: Any, info: MediaPipeline.SpiderInfo + ) -> list[Request]: + return [] - def media_downloaded(self, response, request, info, *, item=None): - return {} + def media_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> FileInfo | Awaitable[FileInfo]: + return cast("FileInfo", {}) - def media_failed(self, failure, request, info): + def media_failed( + self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo + ) -> Failure: failure.raiseException() - def file_path(self, request, response=None, info=None, *, item=None): + def file_path( + self, + request: Request, + response: Response | None = None, + info: MediaPipeline.SpiderInfo | None = None, + *, + item: Any = None, + ) -> str: return "" @@ -48,8 +81,14 @@ class TestBaseMediaPipeline: self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe.open_spider() self.info = self.pipe.spiderinfo + assert crawler.request_fingerprinter is not None self.fingerprint = crawler.request_fingerprinter.fingerprint + @property + def mocked_pipe(self) -> MockedMediaPipeline: + assert isinstance(self.pipe, MockedMediaPipeline) + return self.pipe + def teardown_method(self): for name, signal in vars(signals).items(): if not name.startswith("_"): @@ -121,11 +160,13 @@ class TestBaseMediaPipeline: # When calling the method that caches the Request's result ... self.pipe._cache_result_and_execute_waiters(failure, fp, info) # ... it should store the Twisted Failure ... - assert info.downloaded[fp] == failure + downloaded = info.downloaded[fp] + assert downloaded == failure # ... encapsulating the original FileException ... - assert info.downloaded[fp].value == file_exc + assert isinstance(downloaded, Failure) + assert downloaded.value == file_exc # ... but it should not store the StopIteration exception on its context - context = getattr(info.downloaded[fp].value, "__context__", None) + context = getattr(downloaded.value, "__context__", None) assert context is None def test_default_item_completed(self, caplog: pytest.LogCaptureFixture) -> None: @@ -134,7 +175,7 @@ class TestBaseMediaPipeline: # Check that failures are logged by default fail = Failure(Exception()) - results = [(True, 1), (False, fail)] + results: Any = [(True, 1), (False, fail)] caplog.clear() new_item = self.pipe.item_completed(results, item, self.info) @@ -158,7 +199,7 @@ class TestBaseMediaPipeline: by item_completed(), as they are not download errors.""" item = {"name": "name"} fail = Failure(_MediaRequestFiltered("Filtered offsite request")) - results = [(True, 1), (False, fail)] + results: Any = [(True, 1), (False, fail)] with caplog.at_level(logging.DEBUG): new_item = self.pipe.item_completed(results, item, self.info) @@ -174,29 +215,44 @@ class TestBaseMediaPipeline: class MockedMediaPipeline(UserDefinedPipeline): - def __init__(self, *args, crawler=None, **kwargs): + def __init__(self, *args: Any, crawler: Crawler, **kwargs: Any): super().__init__(*args, crawler=crawler, **kwargs) - self._mockcalled = [] + self._mockcalled: list[str] = [] - def media_to_download(self, request, info, *, item=None): + def media_to_download( + self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None + ) -> Deferred[FileInfo | None] | None: self._mockcalled.append("media_to_download") if "result" in request.meta: return request.meta.get("result") return super().media_to_download(request, info) - def get_media_requests(self, item, info): + def get_media_requests( + self, item: Any, info: MediaPipeline.SpiderInfo + ) -> list[Request]: self._mockcalled.append("get_media_requests") - return item.get("requests") + return item.get("requests") # type: ignore[no-any-return] - def media_downloaded(self, response, request, info, *, item=None): + def media_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> FileInfo | Awaitable[FileInfo]: self._mockcalled.append("media_downloaded") return super().media_downloaded(response, request, info) - def media_failed(self, failure, request, info): + def media_failed( + self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo + ) -> Failure: self._mockcalled.append("media_failed") return super().media_failed(failure, request, info) - def item_completed(self, results, item, info): + def item_completed( + self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + ) -> Any: self._mockcalled.append("item_completed") item = super().item_completed(results, item, info) item["results"] = results @@ -204,7 +260,14 @@ class MockedMediaPipeline(UserDefinedPipeline): class AsyncMediaDownloadedPipeline(MockedMediaPipeline): - async def media_downloaded(self, response, request, info, *, item=None): + async def media_downloaded( # type: ignore[override] + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> FileInfo | Awaitable[FileInfo]: return super().media_downloaded(response, request, info) @@ -212,7 +275,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): pipeline_class = MockedMediaPipeline def _errback(self, result): - self.pipe._mockcalled.append("request_errback") + self.mocked_pipe._mockcalled.append("request_errback") return result @coroutine_test @@ -226,7 +289,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): item = {"requests": req} new_item = await self.pipe.process_item(item) assert new_item["results"] == [(True, {})] - assert self.pipe._mockcalled == [ + assert self.mocked_pipe._mockcalled == [ "get_media_requests", "media_to_download", "media_downloaded", @@ -248,7 +311,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): assert new_item["results"][0][0] is False assert isinstance(new_item["results"][0][1], Failure) assert new_item["results"][0][1].value == exc - assert self.pipe._mockcalled == [ + assert self.mocked_pipe._mockcalled == [ "get_media_requests", "media_to_download", "media_failed", @@ -270,7 +333,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): assert new_item["results"][1][0] is False assert isinstance(new_item["results"][1][1], Failure) assert new_item["results"][1][1].value == exc - m = self.pipe._mockcalled + m = self.mocked_pipe._mockcalled # only once assert m[0] == "get_media_requests" # first hook called assert m.count("get_media_requests") == 1 @@ -294,7 +357,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): # returns iterable of Requests req1 = Request("http://url1") req2 = Request("http://url2") - item = {"requests": iter([req1, req2])} + item = {"requests": iter([req1, req2])} # type: ignore[dict-item] new_item = await self.pipe.process_item(item) assert new_item is item assert self.fingerprint(req1) in self.info.downloaded @@ -304,7 +367,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): async def test_results_are_cached_across_multiple_items(self): rsp1 = Response("http://url1") req1 = Request("http://url1", meta={"response": rsp1}) - item = {"requests": req1} + item: dict[str, Any] = {"requests": req1} new_item = await self.pipe.process_item(item) assert new_item is item assert new_item["results"] == [(True, {})] @@ -335,7 +398,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): new_item = await self.pipe.process_item({"requests": req2}) assert new_item["results"][0][0] is False assert new_item["results"][0][1].value is exc - assert self.pipe._mockcalled.count("media_to_download") == 1 + assert self.mocked_pipe._mockcalled.count("media_to_download") == 1 @coroutine_test async def test_cached_failure_calls_errback(self): @@ -347,13 +410,13 @@ class TestMediaPipeline(TestBaseMediaPipeline): ) def errback(failure): - self.pipe._mockcalled.append("request_errback") + self.mocked_pipe._mockcalled.append("request_errback") return {"recovered": failure.value} req = Request("http://url1", errback=errback) new_item = await self.pipe.process_item({"requests": req}) assert new_item["results"] == [(True, {"recovered": exc})] - assert self.pipe._mockcalled.count("request_errback") == 1 + assert self.mocked_pipe._mockcalled.count("request_errback") == 1 @coroutine_test async def test_results_are_cached_for_requests_of_single_item(self): @@ -362,14 +425,14 @@ class TestMediaPipeline(TestBaseMediaPipeline): req2 = Request( req1.url, meta={"response": Response("http://donot.download.me")} ) - item = {"requests": [req1, req2]} + item: dict[str, Any] = {"requests": [req1, req2]} new_item = await self.pipe.process_item(item) assert new_item is item assert new_item["results"] == [(True, {}), (True, {})] @coroutine_test async def test_wait_if_request_is_downloading(self): - def _check_downloading(response): + def _check_downloading(response: Response) -> Response: fp = self.fingerprint(req1) assert fp in self.info.downloading assert fp in self.info.waiting @@ -398,7 +461,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): item = {"requests": req} new_item = await self.pipe.process_item(item) assert new_item["results"] == [(True, "ITSME")] - assert self.pipe._mockcalled == [ + assert self.mocked_pipe._mockcalled == [ "get_media_requests", "media_to_download", "item_completed", @@ -422,7 +485,9 @@ class TestAsyncMediaDownloaded(TestMediaPipeline): class TestMediaPipelineAllowRedirectSettings: - def _assert_request_no3xx(self, pipeline_class, settings): + def _assert_request_no3xx( + self, pipeline_class: type[MediaPipeline], settings: dict[str, Any] + ) -> None: pipe = pipeline_class(crawler=get_crawler(None, settings)) request = Request("http://url") pipe._modify_media_request(request) @@ -477,7 +542,7 @@ class TestBuildFromCrawler: self._init_called = True @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Pipeline: settings = crawler.settings store_uri = settings["FILES_STORE"] o = cls(store_uri, settings=settings, crawler=crawler) @@ -493,9 +558,10 @@ class TestBuildFromCrawler: def test_has_from_crawler(self): class Pipeline(UserDefinedPipeline): _from_crawler_called = False + store_uri: str @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Pipeline: settings = crawler.settings o = super().from_crawler(crawler) o._from_crawler_called = True @@ -509,7 +575,9 @@ class TestBuildFromCrawler: class MediaFailedNonePipeline(MockedMediaPipeline): - def media_failed(self, failure, request, info): + def media_failed( # type: ignore[override] + self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo + ) -> None: self._mockcalled.append("media_failed") @@ -524,7 +592,7 @@ class TestMediaFailedNone(TestBaseMediaPipeline): req = Request("http://url1", meta={"response": Exception("foo")}) new_item = await self.pipe.process_item({"requests": req}) assert new_item["results"] == [(True, None)] - assert self.pipe._mockcalled == [ + assert self.mocked_pipe._mockcalled == [ "get_media_requests", "media_to_download", "media_failed", @@ -533,7 +601,9 @@ class TestMediaFailedNone(TestBaseMediaPipeline): class MediaFailedFailurePipeline(MockedMediaPipeline): - def media_failed(self, failure, request, info): + def media_failed( + self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo + ) -> Failure: self._mockcalled.append("media_failed") return failure # deprecated @@ -544,7 +614,7 @@ class TestMediaFailedFailure(TestBaseMediaPipeline): pipeline_class = MediaFailedFailurePipeline def _errback(self, result): - self.pipe._mockcalled.append("request_errback") + self.mocked_pipe._mockcalled.append("request_errback") return result @coroutine_test @@ -565,7 +635,7 @@ class TestMediaFailedFailure(TestBaseMediaPipeline): assert new_item["results"][0][0] is False assert isinstance(new_item["results"][0][1], Failure) assert new_item["results"][0][1].value == exc - assert self.pipe._mockcalled == [ + assert self.mocked_pipe._mockcalled == [ "get_media_requests", "media_to_download", "media_failed", diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index e65b5bc32..4b34946ae 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -47,7 +47,7 @@ class DeferredPipeline: return succeed(None) def process_item(self, item): - d = Deferred() + d: Deferred[Any] = Deferred() d.addCallback(self.cb) d.callback(item) return d @@ -55,7 +55,7 @@ class DeferredPipeline: class AsyncDefPipeline: async def process_item(self, item): - d = Deferred() + d: Deferred[Any] = Deferred() call_later(0, d.callback, None) await maybe_deferred_to_future(d) item["pipeline_passed"] = True @@ -64,7 +64,7 @@ class AsyncDefPipeline: class AsyncDefAsyncioPipeline: async def process_item(self, item): - d = Deferred() + d: Deferred[Any] = Deferred() loop = asyncio.get_event_loop() loop.call_later(0, d.callback, None) await deferred_to_future(d) @@ -75,12 +75,12 @@ class AsyncDefAsyncioPipeline: class AsyncDefNotAsyncioPipeline: async def process_item(self, item): - d1 = Deferred() + d1: Deferred[Any] = Deferred() from twisted.internet import reactor reactor.callLater(0, d1.callback, None) await d1 - d2 = Deferred() + d2: Deferred[Any] = Deferred() reactor.callLater(0, d2.callback, None) await maybe_deferred_to_future(d2) item["pipeline_passed"] = True @@ -120,6 +120,8 @@ class OpenSpiderExceptionAsyncPipeline: class ItemSpider(Spider): name = "itemspider" + mockserver: MockServer + async def start(self): yield Request(self.mockserver.url("/status?n=200")) From bee31890a23f403ca31b084dc3ec73c2c99eec43 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 19:56:03 +0200 Subject: [PATCH 092/111] Report exceptions from Spider.start() (#7884) --- docs/topics/signals.rst | 13 ++++-- docs/topics/spiders.rst | 32 +++++++++++++++ docs/topics/stats.rst | 17 +++++--- scrapy/core/engine.py | 23 ++++++++++- scrapy/exceptions.py | 8 +++- tests/test_crawler_subprocess.py | 2 +- tests/test_engine_loop.py | 69 ++++++++++++++++++++++++++++++++ 7 files changed, 152 insertions(+), 12 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index f060710a4..acbf7ddef 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -338,15 +338,22 @@ spider_error .. signal:: spider_error .. function:: spider_error(failure, response, spider) - Sent when a spider callback generates an error (i.e. raises an exception). + Sent when a spider callback or the :meth:`~scrapy.Spider.start` method of a + spider generates an error (i.e. raises an exception). + + .. versionchanged:: VERSION + Exceptions from :meth:`~scrapy.Spider.start` are also reported, see + :ref:`start-error`. This signal does not support :ref:`asynchronous handlers `. :param failure: the exception raised :type failure: twisted.python.failure.Failure - :param response: the response being processed when the exception was raised - :type response: :class:`~scrapy.http.Response` object + :param response: the response being processed when the exception was + raised, or ``None`` if the exception came from + :meth:`~scrapy.Spider.start`. + :type response: :class:`~scrapy.http.Response` | ``None`` :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e68c208b7..95c80d5dc 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -411,6 +411,38 @@ scheduled requests: await self.crawler.signals.wait_for(signals.scheduler_empty) yield item_or_request +.. _start-error: + +Handling start errors +--------------------- + +An exception raised by :meth:`~scrapy.Spider.start` ends its iteration, so any +remaining start items and requests are never sent. Scrapy logs the exception, +sends the :signal:`spider_error` signal, and, once the already scheduled +requests are done, closes the spider with the ``start_error`` +:stat:`finish_reason`. + +.. versionchanged:: VERSION + The close reason used to be ``finished``, and neither the + :signal:`spider_error` signal nor the :stat:`spider_exceptions/count` stat + reported the exception. + +To keep the iteration going, catch the exception yourself: + +.. code-block:: python + + async def start(self): + for url in self.start_urls: + try: + request = Request(url) + except ValueError: + self.logger.exception(f"Skipping start URL {url}") + else: + yield request + +To stop the crawl instead, and choose your own :stat:`finish_reason`, raise +:exc:`~scrapy.exceptions.CloseSpider`. + .. _builtin-spiders: Generic Spiders diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index b558c1cf2..6fb7783d1 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -301,6 +301,10 @@ one per actual value of the placeholder. - ``shutdown``: the crawl was interrupted, e.g. by a system signal such as ``SIGINT`` (:kbd:`Ctrl-C`). + - ``start_error``: :meth:`~scrapy.Spider.start` raised an exception, so + some :ref:`start requests ` may never have been sent, + see :ref:`start-error`. + Third-party components and your own code may use any other reason, e.g. by raising :exc:`~scrapy.exceptions.CloseSpider` with it. @@ -721,18 +725,21 @@ one per actual value of the placeholder. .. stat:: spider_exceptions/count ``spider_exceptions/count`` - Number of unhandled exceptions raised by spider callbacks. + Number of unhandled exceptions raised by spider callbacks or by + :meth:`~scrapy.Spider.start`. - Set by the :ref:`scraper `. + Set by the :ref:`engine ` and the :ref:`scraper + `. .. stat:: spider_exceptions/{exception} ``spider_exceptions/{exception}`` - Number of unhandled exceptions raised by spider callbacks, per exception, - where ``{exception}`` is the class name of the exception, e.g. + Same as :stat:`spider_exceptions/count`, per exception, where + ``{exception}`` is the class name of the exception, e.g. ``spider_exceptions/ValueError``. - Set by the :ref:`scraper `. + Set by the :ref:`engine ` and the :ref:`scraper + `. .. stat:: start_time diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index efac5f71c..b7a1e0fcd 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -125,6 +125,9 @@ class ExecutionEngine: ] = spider_closed_callback self.start_time: float | None = None self._start: AsyncIterator[Any] | None = None + # Whether Spider.start() raised, i.e. some start items or requests may + # never have reached the engine. + self._start_error: bool = False self._closewait: Deferred[None] | None = None self._start_request_processing_awaitable: ( asyncio.Future[None] | Deferred[None] | None @@ -277,13 +280,30 @@ class ExecutionEngine: item_or_request = await anext(self._start) except StopAsyncIteration: self._start = None + except CloseSpider as exception: + self._start = None + _schedule_coro( + self.close_spider_async(reason=exception.reason or "cancelled") + ) except Exception as exception: self._start = None + self._start_error = True exception_traceback = format_exc() logger.error( f"Error while reading start items and requests: {exception}.\n{exception_traceback}", exc_info=True, ) + self.signals.send_catch_log( + signal=signals.spider_error, + failure=Failure(), + response=None, + spider=self.spider, + ) + assert self.crawler.stats + self.crawler.stats.inc_value("spider_exceptions/count") + self.crawler.stats.inc_value( + f"spider_exceptions/{type(exception).__name__}" + ) else: if not self.spider: return # spider already closed @@ -579,7 +599,8 @@ class ExecutionEngine: if DontCloseSpider in detected_ex: return if self.spider_is_idle(): - ex = detected_ex.get(CloseSpider, CloseSpider(reason="finished")) + default_reason = "start_error" if self._start_error else "finished" + ex = detected_ex.get(CloseSpider, CloseSpider(reason=default_reason)) assert isinstance(ex, CloseSpider) # typing _schedule_coro(self.close_spider_async(reason=ex.reason)) diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index ccaccddf2..fa5bc1c1e 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -56,8 +56,12 @@ class DontCloseSpider(Exception): class CloseSpider(Exception): - """Raised from a :ref:`spider callback ` to request the - spider to be closed/stopped. + """Raised from a :ref:`spider callback ` or from + :meth:`~scrapy.Spider.start` to request the spider to be closed/stopped. + + .. versionchanged:: VERSION + Raising it from :meth:`~scrapy.Spider.start` closes the spider, instead + of being reported as a start error. *reason* is a string with the reason for closing. diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index 240482586..defee1041 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -400,7 +400,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): def test_reactorless_import_hook(self) -> None: log = self.run_script("reactorless_import_hook.py") assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log + assert "Spider closed (start_error)" in log assert "ImportError: Import of twisted.internet.reactor is forbidden" in log def test_reactorless_import_hook_uninstall(self) -> None: diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 14ec3d184..1ecf8b8de 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.core.scheduler import BaseScheduler +from scrapy.exceptions import CloseSpider from scrapy.utils.asyncio import call_later, sleep from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer @@ -140,6 +141,74 @@ class TestMain: assert crawler.stats.get_value("finish_reason") == "shutdown" assert not actual_urls + @coroutine_test + async def test_start_error(self, caplog: pytest.LogCaptureFixture) -> None: + class TestSpider(Spider): + name = "test" + + async def start(self): + yield Request("data:,a") + raise ValueError + + def parse(self, response): + pass + + actual_urls = [] + errors = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + def track_error(failure, response, spider): + errors.append((failure, response)) + + settings = {"SCHEDULER": MemoryScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + crawler.signals.connect(track_error, signals.spider_error) + + caplog.clear() + with caplog.at_level(ERROR): + await crawler.crawl_async() + + # The requests yielded before the exception are still crawled. + assert actual_urls == ["data:,a"] + assert len(caplog.records) == 1 + assert len(errors) == 1 + failure, response = errors[0] + assert isinstance(failure.value, ValueError) + assert response is None + assert crawler.stats + assert crawler.stats.get_value("finish_reason") == "start_error" + assert crawler.stats.get_value("spider_exceptions/count") == 1 + assert crawler.stats.get_value("spider_exceptions/ValueError") == 1 + + @coroutine_test + async def test_close_spider_from_start( + self, caplog: pytest.LogCaptureFixture + ) -> None: + class TestSpider(Spider): + name = "test" + + async def start(self): + yield Request("data:,a") + raise CloseSpider("my_reason") + + def parse(self, response): + pass + + settings = {"SCHEDULER": MemoryScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + + caplog.clear() + with caplog.at_level(ERROR): + await crawler.crawl_async() + + assert not caplog.records + assert crawler.stats + assert crawler.stats.get_value("finish_reason") == "my_reason" + assert crawler.stats.get_value("spider_exceptions/count") is None + class TestRequestSendOrder: seconds = 0.1 # increase if flaky From aadfd153275258262ec94e531f81757a3520bd39 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 19:58:35 +0200 Subject: [PATCH 093/111] Do not mangle log messages when a log formatter returns empty or tuple args (#7936) --- scrapy/utils/log.py | 14 +++++++++----- tests/test_utils_log.py | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 7645b235e..ee357c031 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -245,7 +245,7 @@ class LogCounterHandler(logging.Handler): def logformatter_adapter( logkws: LogFormatterResult, -) -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]: +) -> tuple[Any, ...]: """ Helper that takes the dictionary output from the methods in LogFormatter and adapts it into a tuple of positional arguments for logger.log calls. @@ -253,10 +253,14 @@ def logformatter_adapter( level = logkws.get("level", logging.INFO) message = logkws.get("msg") or "" - # NOTE: This also handles 'args' being an empty dict, that case doesn't - # play well in logger.log calls - args = cast("dict[str, Any]", logkws) if not logkws.get("args") else logkws["args"] - + args = logkws.get("args") + # logging interpolates the message whenever it receives any positional + # argument, so empty args are left out. Tuple args become one positional + # argument each, while a dict is a single positional argument. + if not args: + return (level, message) + if isinstance(args, tuple): + return (level, message, *args) return (level, message, args) diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 7f5301387..42b2b95fd 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -5,7 +5,7 @@ import logging import re import sys from io import StringIO -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import pytest from twisted.python.failure import Failure @@ -16,6 +16,7 @@ from scrapy.utils.log import ( StreamLogger, TopLevelFormatter, failure_to_exc_info, + logformatter_adapter, ) from scrapy.utils.test import get_crawler from tests.spiders import LogSpider @@ -24,6 +25,7 @@ if TYPE_CHECKING: from collections.abc import Generator, Mapping, MutableMapping from scrapy.crawler import Crawler + from scrapy.logformatter import LogFormatterResult class TestFailureToExcInfo: @@ -311,3 +313,36 @@ class TestLoggingWithExtra: assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] + + +class TestLogformatterAdapter: + @staticmethod + def _log(caplog: pytest.LogCaptureFixture, logkws: LogFormatterResult) -> str: + with caplog.at_level(logging.INFO): + logging.getLogger(__name__).log(*logformatter_adapter(logkws)) + return caplog.records[-1].getMessage() + + @pytest.mark.parametrize("args", [None, {}, ()]) + def test_empty_args( + self, + caplog: pytest.LogCaptureFixture, + args: dict[str, Any] | tuple[Any, ...] | None, + ) -> None: + logkws = cast( + "LogFormatterResult", + {"level": logging.INFO, "msg": "90% done", "args": args}, + ) + assert self._log(caplog, logkws) == "90% done" + + @pytest.mark.parametrize( + ("msg", "args"), + [("%(pct)d%% done", {"pct": 90}), ("%d%% done", (90,))], + ) + def test_args( + self, + caplog: pytest.LogCaptureFixture, + msg: str, + args: dict[str, Any] | tuple[Any, ...], + ) -> None: + logkws: LogFormatterResult = {"level": logging.INFO, "msg": msg, "args": args} + assert self._log(caplog, logkws) == "90% done" From fc082aa914cfb2a610d672fb39d5beaba47b0a7d Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 20:01:31 +0200 Subject: [PATCH 094/111] Clarify the docs about reactor settings (#7880) --- docs/topics/settings.rst | 46 ++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index de665f42e..de74ad5cf 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -69,9 +69,10 @@ Example:: precedence and override the project ones. .. note:: :ref:`Pre-crawler settings ` cannot be defined - per spider, and :ref:`reactor settings ` should not have - a different value per spider when :ref:`running multiple spiders in the - same process `. + per spider, and :ref:`reactor settings ` and + :ref:`logging settings ` are subject to restrictions when + :ref:`running multiple spiders in the same process + `. One way to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute: @@ -329,32 +330,41 @@ Reactor settings **Reactor settings** are settings tied to the :doc:`Twisted reactor `. -These settings can be defined from a spider. However, because only 1 reactor -can be used per process, these settings cannot use a different value per spider -when :ref:`running multiple spiders in the same process -`. +Because only 1 reactor can be used per process, these settings cannot use a +different value per spider when :ref:`running multiple spiders in the same +process `. -In general, if different spiders define different values, the first defined -value is used. However, if two spiders request a different reactor, an -exception is raised. - -These settings are: +These settings are used upon installing the reactor: - :setting:`ASYNCIO_EVENT_LOOP` (not possible to set per-spider when using :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below) +- :setting:`TWISTED_REACTOR` (ignored when using + :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below) + +They can be :ref:`set from a spider `, but only the values +from the first spider that runs are used, since that is when the reactor is +installed. If a later spider asks for a different reactor or a different event +loop, an exception is raised. With +:class:`~scrapy.crawler.CrawlerRunner` and +:class:`~scrapy.crawler.AsyncCrawlerRunner` the reactor must be installed +beforehand, so these settings are only used to check that the installed reactor +and event loop match them. + +These settings are applied when starting the reactor: + - :setting:`TWISTED_DNS_RESOLVER` and settings used by the corresponding component, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE` and :setting:`DNS_TIMEOUT` for the default one. - :setting:`REACTOR_THREADPOOL_MAXSIZE` -- :setting:`TWISTED_REACTOR` (ignored when using - :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below) - -:setting:`ASYNCIO_EVENT_LOOP` and :setting:`TWISTED_REACTOR` are used upon -installing the reactor. The rest of the settings are applied when starting -the reactor. +They are read from the settings of the +:class:`~scrapy.crawler.CrawlerProcess` or +:class:`~scrapy.crawler.AsyncCrawlerProcess` object, so setting them from a +spider or an :ref:`add-on ` has no effect. They are ignored +altogether when using :class:`~scrapy.crawler.CrawlerRunner` or +:class:`~scrapy.crawler.AsyncCrawlerRunner`, which do not start the reactor. There is an additional restriction for :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` when using From ba29ce8d3bca5d525d39cd5f05c97e8416f43857 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 20:02:44 +0200 Subject: [PATCH 095/111] Add a docs section on using download_async() from a downloader middleware (#7872) --- docs/topics/downloader-middleware.rst | 55 +++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 2d6ba0cf3..965108214 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -156,6 +156,61 @@ defines one or more of these methods: :param exception: the raised exception :type exception: an ``Exception`` object +.. _mw-download: + +Downloading a request from a downloader middleware +================================================== + +A downloader middleware can download a request of its own while it processes +another one, e.g. to fetch something that the request it is processing needs. +The built-in :ref:`robots.txt middleware ` does that: it +holds each request while it downloads the ``robots.txt`` file of its website. + +Use :meth:`crawler.engine.download_async() +` for that: + +.. code-block:: python + + from scrapy import Request + from scrapy.http.request import NO_CALLBACK + + + class TokenMiddleware: + def __init__(self, crawler): + self.crawler = crawler + self.token = None + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + async def process_request(self, request): + if request.meta.get("dont_obey_robotstxt"): + return + if self.token is None: + response = await self.crawler.engine.download_async( + Request( + "https://example.com/token", + callback=NO_CALLBACK, + meta={"dont_obey_robotstxt": True}, + ) + ) + self.token = response.text + request.headers["Authorization"] = self.token + +Requests that you download this way go through the downloader middleware chain +as well, including your own middleware and the :ref:`robots.txt middleware +`, which holds a request until the ``robots.txt`` file of +its website arrives. Be careful not to introduce deadlocks: a request that you +download must not end up waiting for the request that is waiting for it. Hence +:reqmeta:`dont_obey_robotstxt` above, which makes both middlewares let the token +request through. + +While the first token response is in transit, ``process_request`` runs for other +requests as well, and the middleware above downloads a token for each of them. +Cache the task that downloads the token, and not only its result, to download +the token only once. + .. _topics-downloader-middleware-ref: Built-in downloader middleware reference From d41bfaec05d90fbca2f2c7f1652741fcf7a06a05 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 20:05:43 +0200 Subject: [PATCH 096/111] Document how to disallow subdomains of allowed domains (#7903) --- docs/topics/downloader-middleware.rst | 35 +--------------- scrapy/downloadermiddlewares/offsite.py | 48 +++++++++++++++++++++- tests/test_downloadermiddleware_offsite.py | 16 ++++++++ 3 files changed, 65 insertions(+), 34 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 965108214..dec9904d2 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -799,40 +799,9 @@ OffsiteMiddleware .. module:: scrapy.downloadermiddlewares.offsite :synopsis: Offsite Middleware -.. class:: OffsiteMiddleware +.. autoclass:: OffsiteMiddleware - .. versionadded:: 2.11.2 - - Filters out Requests for URLs outside the domains covered by the spider. - - This middleware filters out every request whose host names aren't in the - spider's :attr:`~scrapy.Spider.allowed_domains` attribute. - All subdomains of any domain in the list are also allowed. - E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` - but not ``www2.example.com`` nor ``example.com``. - - When your spider returns a request for a domain not belonging to those - covered by the spider, this middleware will log a debug message similar to - this one:: - - DEBUG: Filtered offsite request to 'offsite.example': - - To avoid filling the log with too much noise, it will only print one of - these messages for each new domain filtered. So, for example, if another - request for ``offsite.example`` is filtered, no log message will be - printed. But if a request for ``other.example`` is filtered, a message - will be printed (but only for the first request filtered). - - If the spider doesn't define an - :attr:`~scrapy.Spider.allowed_domains` attribute, or the - attribute is empty, the offsite middleware will allow all requests. - - .. reqmeta:: allow_offsite - - If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to - ``True`` or :attr:`Request.meta ` has ``allow_offsite`` - set to ``True``, then the OffsiteMiddleware will allow the request even if - its domain is not listed in allowed domains. + .. automethod:: should_follow RedirectMiddleware ------------------ diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 28c0e09cb..0d5c9e4c9 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -21,6 +21,36 @@ logger = logging.getLogger(__name__) class OffsiteMiddleware: + """Filter out requests for URLs outside the domains covered by the spider. + + .. versionadded:: 2.11.2 + + A request is allowed if its host name is in the + :attr:`~scrapy.Spider.allowed_domains` attribute of the spider, or is a + subdomain of one of those domains. E.g. ``www.example.org`` also allows + ``bob.www.example.org``, but neither ``www2.example.org`` nor + ``example.org``. See :meth:`should_follow` to use a different policy. + + If the spider does not define :attr:`~scrapy.Spider.allowed_domains`, or + the attribute is empty, every request is allowed. + + Filtered requests are logged as follows:: + + DEBUG: Filtered offsite request to 'offsite.example': + + Only the first request filtered for a given domain is logged, to keep the + log readable. + + .. reqmeta:: allow_offsite + + allow_offsite + ------------- + + Requests with the ``allow_offsite`` :attr:`~scrapy.Request.meta` key set to + ``True``, or with :attr:`~scrapy.Request.dont_filter` set to ``True``, are + allowed regardless of their host name. + """ + crawler: Crawler host_regex: re.Pattern[str] @@ -72,6 +102,23 @@ class OffsiteMiddleware: raise IgnoreRequest(f"Filtered offsite request to {domain!r}") def should_follow(self, request: Request, spider: Spider) -> bool: + """Return ``True`` if *request* is on site, ``False`` if it must be + filtered out. + + Override this method to implement a different offsite policy. For + example, to allow the domains in + :attr:`~scrapy.Spider.allowed_domains` but none of their subdomains: + + .. code-block:: python + + from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware + from scrapy.utils.httpobj import urlparse_cached + + + class RootOnlyOffsiteMiddleware(OffsiteMiddleware): + def should_follow(self, request, spider): + return urlparse_cached(request).hostname in spider.allowed_domains + """ self._update_host_regex(spider) regex = self.host_regex # hostname can be None for wrong urls (like javascript links) @@ -79,7 +126,6 @@ class OffsiteMiddleware: return bool(regex.search(host)) def get_host_regex(self, spider: Spider) -> re.Pattern[str]: - """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, "allowed_domains", None) if not allowed_domains: return re.compile("") # allow all by default diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index bab89814f..1f91dcd4b 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -6,6 +6,7 @@ import pytest from scrapy import Request, Spider from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware from scrapy.exceptions import IgnoreRequest +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.test import get_crawler UNSET = object() @@ -237,6 +238,21 @@ def test_repeated_offsite_domain(): assert crawler.stats.get_value("offsite/filtered") == 2 +def test_should_follow_override(): + class RootOnlyOffsiteMiddleware(OffsiteMiddleware): + def should_follow(self, request: Request, spider: Spider) -> bool: + allowed_domains: list[str] = getattr(spider, "allowed_domains", []) + return urlparse_cached(request).hostname in allowed_domains + + crawler = get_crawler(Spider) + crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"]) + mw = RootOnlyOffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(crawler.spider) + assert mw.process_request(Request("https://example.com/1")) is None + with pytest.raises(IgnoreRequest): + mw.process_request(Request("https://www.example.com/1")) + + def test_ignore_request_reason(): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"]) From 5270f3cf99d10877f9b4c044e8a21de883c94ea5 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 20:07:10 +0200 Subject: [PATCH 097/111] Add a depth_reset request metadata key (#7913) --- docs/topics/spider-middleware.rst | 20 +------------------- scrapy/spidermiddlewares/depth.py | 26 +++++++++++++++++++++++++- tests/test_spidermiddleware_depth.py | 13 +++++++++++++ 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index db85906c1..1c9ee0c77 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -228,25 +228,7 @@ DepthMiddleware .. module:: scrapy.spidermiddlewares.depth :synopsis: Depth Spider Middleware -.. class:: DepthMiddleware - - DepthMiddleware is used for tracking the depth of each Request inside the - site being scraped. It works by setting ``request.meta['depth'] = 0`` whenever - there is no value previously set (usually just the first Request) and - incrementing it by 1 otherwise. - - It can be used to limit the maximum depth to scrape, control Request - priority based on their depth, and things like that. - - The :class:`DepthMiddleware` can be configured through the following - settings (see the settings documentation for more info): - - * :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to - crawl for any site. If zero, no limit will be imposed. - * :setting:`DEPTH_STATS_VERBOSE` - Whether to collect the number of - requests for each depth. - * :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on - their depth. +.. autoclass:: DepthMiddleware HttpErrorMiddleware ------------------- diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 0131b62e7..1781543e3 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -28,6 +28,27 @@ logger = logging.getLogger(__name__) class DepthMiddleware(BaseSpiderMiddleware): + """Track the depth of each request within the site being scraped, setting + ``request.meta["depth"]`` to 0 when there is no value previously set + (usually just the first request) and incrementing it by 1 otherwise. + + It can be used to limit the maximum depth to scrape, control request + priority based on their depth, and things like that, through the + :setting:`DEPTH_LIMIT`, :setting:`DEPTH_STATS_VERBOSE` and + :setting:`DEPTH_PRIORITY` settings. + + .. reqmeta:: depth_reset + + depth_reset + ----------- + + .. versionadded:: VERSION + + :attr:`~scrapy.Request.meta` key that, set to ``True``, gives a request + depth 0 instead of the depth of its source response plus 1, e.g. to keep + :setting:`DEPTH_LIMIT` from applying across a domain change. + """ + crawler: Crawler def __init__( # pylint: disable=super-init-not-called @@ -87,10 +108,13 @@ class DepthMiddleware(BaseSpiderMiddleware): def get_processed_request( self, request: Request, response: Response | None ) -> Request | None: + # Consumed here so that it cannot reach response.meta and, from there, + # spread to further requests through a meta copy. + depth_reset = request.meta.pop("depth_reset", False) if response is None: # start requests return request - depth = response.meta["depth"] + 1 + depth = 0 if depth_reset else response.meta["depth"] + 1 request.meta["depth"] = depth if self.prio: request.priority -= depth * self.prio diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index 2aa76195e..bdacd7287 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -60,6 +60,19 @@ def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> No assert rdm == 1 +def test_depth_reset(mw: DepthMiddleware, stats: StatsCollector) -> None: + resp = Response("https://example.com") + resp.request = Request("https://example.com", meta={"depth": 5}) + result = [Request("https://example.com", meta={"depth_reset": True})] + + out = list(mw.process_spider_output(resp, result)) + + assert out == result + assert out[0].meta["depth"] == 0 + assert "depth_reset" not in out[0].meta + assert stats.get_value("request_depth_count/0") == 1 + + def test_process_spider_output_no_response( mw: DepthMiddleware, stats: StatsCollector ) -> None: From ae68786210b99240b269125673704754a282ff5e Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 20:08:59 +0200 Subject: [PATCH 098/111] Document how to customize media pipeline file names from responses (#7909) --- docs/topics/media-pipeline.rst | 35 +++++++++++++++++-- tests/test_pipeline_files.py | 64 +++++++++++++++++++++++++++++++--- 2 files changed, 93 insertions(+), 6 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index b16066d0c..576feae7e 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -178,6 +178,37 @@ By overriding ``file_path`` like this: For more information about the ``file_path`` method, see :ref:`topics-media-pipeline-override`. +.. _file-naming-response: + +Naming files after the response +------------------------------- + +``file_path`` also receives the ``response``, which allows naming files after +response data. For example, to determine the file extension from the +``Content-Type`` header, for URLs that do not end in a file name: + +.. code-block:: python + + import mimetypes + + from scrapy.pipelines.files import FilesPipeline + + + class ContentTypeFilesPipeline(FilesPipeline): + def file_path(self, request, response=None, info=None, *, item=None): + path = super().file_path(request, response, info, item=item) + if response is None: + return path + content_type = response.headers["Content-Type"].decode() + return path + (mimetypes.guess_extension(content_type) or "") + +This requires setting :setting:`FILES_EXPIRES` to ``0``. To find out whether a +file has already been downloaded, Scrapy calls ``file_path`` before the +download, with ``response`` set to ``None``, and checks the age of the file at +the resulting path. A path that depends on the response can never match that +check, and :setting:`FILES_EXPIRES` set to ``0`` disables it, at the cost of +downloading every file on every run. + .. _topics-supported-storage: Supported Storage @@ -543,7 +574,7 @@ See here the methods that you can override in your custom Files Pipeline: return "files/" + PurePosixPath(urlparse_cached(request).path).name Similarly, you can use the ``item`` to determine the file path based on some item - property. + property, or the ``response``, see :ref:`file-naming-response`. By default the :meth:`file_path` method returns ``full/.``. @@ -693,7 +724,7 @@ See here the methods that you can override in your custom Images Pipeline: return "files/" + PurePosixPath(urlparse_cached(request).path).name Similarly, you can use the ``item`` to determine the file path based on some item - property. + property, or the ``response``, see :ref:`file-naming-response`. By default the :meth:`file_path` method returns ``full/.``. diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 3a8a3d0f7..f40619933 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,6 +1,7 @@ import base64 import dataclasses import logging +import mimetypes import random import re import time @@ -96,8 +97,14 @@ class TestFilesPipeline: def teardown_method(self): rmtree(self.tempdir) - def _create_pipeline(self, pipeline_cls: type[FilesPipeline]) -> FilesPipeline: - crawler = get_crawler(DefaultSpider, {"FILES_STORE": self.tempdir}) + def _create_pipeline( + self, + pipeline_cls: type[FilesPipeline], + settings: dict[str, Any] | None = None, + ) -> FilesPipeline: + crawler = get_crawler( + DefaultSpider, {"FILES_STORE": self.tempdir, **(settings or {})} + ) crawler.spider = crawler._create_spider() crawler.engine = MagicMock(download_async=mocked_download_func) pipeline = pipeline_cls.from_crawler(crawler) @@ -394,6 +401,47 @@ class TestFilesPipeline: request = Request("http://example.com") assert file_path(request, item=item) == "full/path-to-store-file" + @coroutine_test + async def test_file_path_from_response(self) -> None: + """file_path() may build the path out of the response, e.g. to get the + file extension from a response header, as long as FILES_EXPIRES is 0 to + disable the up-to-date check, which runs before the download and hence + cannot reach the same path.""" + + class ContentTypeFilesPipeline(FilesPipeline): + def file_path(self, request, response=None, info=None, *, item=None): + path = super().file_path(request, response, info, item=item) + if response is None: + return path + content_type = response.headers["Content-Type"].decode() + return path + (mimetypes.guess_extension(content_type) or "") + + item_url = "http://example.com/download?id=1" + item = _create_item_with_files(item_url) + pipeline = self._create_pipeline(ContentTypeFilesPipeline, {"FILES_EXPIRES": 0}) + request = _prepare_request_object( + item_url, headers={"Content-Type": "application/pdf"} + ) + with ( + mock.patch.object(FilesPipeline, "inc_stats", return_value=True), + # A fresh file at the response-less path is ignored thanks to + # FILES_EXPIRES being 0. + mock.patch.object( + FSFilesStore, + "stat_file", + return_value={"checksum": "abc", "last_modified": time.time()}, + ), + mock.patch.object( + FilesPipeline, "get_media_requests", return_value=[request] + ), + ): + result = await pipeline.process_item(item) + + file_info = result["files"][0] + assert file_info["status"] == "downloaded" + assert file_info["path"].endswith(".pdf") + assert (Path(self.tempdir) / file_info["path"]).read_bytes() == b"data" + def test_media_failed_filtered_request( self, caplog: pytest.LogCaptureFixture ) -> None: @@ -1133,10 +1181,18 @@ def _create_item_with_files(*files: str) -> ItemWithFiles: return item -def _prepare_request_object(item_url: str, flags: list[str] | None = None) -> Request: +def _prepare_request_object( + item_url: str, + flags: list[str] | None = None, + headers: dict[str, str] | None = None, +) -> Request: return Request( item_url, - meta={"response": Response(item_url, status=200, body=b"data", flags=flags)}, + meta={ + "response": Response( + item_url, status=200, body=b"data", flags=flags, headers=headers + ) + }, ) From 09c918115bb43d4f5246be2d2a6e5b719182a5c1 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 10 Aug 2026 08:43:57 +0200 Subject: [PATCH 099/111] Document response parsing memory use in the security page (#7930) --- docs/topics/security.rst | 71 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/topics/security.rst b/docs/topics/security.rst index 2ca270045..5348aae23 100644 --- a/docs/topics/security.rst +++ b/docs/topics/security.rst @@ -36,6 +36,77 @@ their input in an unsafe way, such as :func:`eval`, :func:`exec`, or :func:`pickle.loads`, and be careful when writing response data to paths derived from the response itself. +.. _security-response-size: + +Memory use when parsing responses +================================= + +Parsing a response with :ref:`selectors ` builds an in-memory +tree of the whole response body, which takes several times as much memory as +the body itself. Scrapy parses without the size limits that libxml2 applies by +default, so the size of that tree is bound only by the size of the response, as +controlled by :setting:`DOWNLOAD_MAXSIZE` (default: 1 GiB). + +XML entities are left unresolved, so the tree stays proportional to the +response body even for input crafted as an `XML bomb +`_. A server can still +make a crawler allocate a lot of memory by returning a very large response, +though, so if you know the size of the responses you care about, lower the +limit: + +.. code-block:: python + + DOWNLOAD_MAXSIZE = 32 * 1024 * 1024 # 32 MiB + +* **Pro:** a server cannot make the crawler allocate more memory than the limit + allows, whether by returning a large response or by crafting one that is + expensive to parse. + +* **Con:** you can no longer scrape sites that legitimately serve responses + above the limit, as those responses are dropped. + +.. _security-parser-limits: + +Parser limits +------------- + +The limits that libxml2 applies by default, such as 256 nesting levels and +10 MB per text node, can be restored by overriding +:attr:`~scrapy.http.TextResponse.selector` in a response subclass and swapping +responses in a :ref:`downloader middleware `: + +.. code-block:: python + + from functools import cached_property + + from scrapy import Selector + from scrapy.http import HtmlResponse + + + class LimitedHtmlResponse(HtmlResponse): + @cached_property + def selector(self): + return Selector(self, huge_tree=False) + + + class LimitedParsingMiddleware: + def process_response(self, request, response, spider): + if isinstance(response, HtmlResponse): + return response.replace(cls=LimitedHtmlResponse) + return response + +Do the same with :class:`~scrapy.http.XmlResponse` if you also parse XML. + +These limits apply per node, so :setting:`DOWNLOAD_MAXSIZE` remains your bound +on total memory: a response made of many small elements is parsed in full and +uses as much memory either way. + +* **Pro:** deeply nested responses, and responses with very large individual + nodes, become cheaper to parse. + +* **Con:** parsing stops at those limits without raising, so a legitimate page + that exceeds them yields incomplete data and no error. + TLS connections =============== From 508bd7faec44975ac774aeff8bb5f4a3f57a0be5 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 10 Aug 2026 09:07:34 +0200 Subject: [PATCH 100/111] Type late Crawler attributes as always set instead of None (#7882) --- docs/topics/api.rst | 7 ++ scrapy/commands/parse.py | 1 - scrapy/core/engine.py | 24 +++---- scrapy/core/scraper.py | 11 +--- scrapy/crawler.py | 78 ++++++++++++++++++----- scrapy/downloadermiddlewares/httpcache.py | 1 - scrapy/downloadermiddlewares/offsite.py | 1 - scrapy/downloadermiddlewares/retry.py | 1 - scrapy/downloadermiddlewares/robotstxt.py | 19 ++---- scrapy/downloadermiddlewares/stats.py | 1 - scrapy/dupefilters.py | 2 - scrapy/extensions/closespider.py | 2 - scrapy/extensions/corestats.py | 1 - scrapy/extensions/debug.py | 1 - scrapy/extensions/feedexport.py | 1 - scrapy/extensions/httpcache.py | 2 - scrapy/extensions/logstats.py | 1 - scrapy/extensions/memdebug.py | 1 - scrapy/extensions/memusage.py | 26 +++----- scrapy/extensions/periodic_log.py | 1 - scrapy/extensions/statsmailer.py | 1 - scrapy/extensions/telnet.py | 1 - scrapy/extensions/throttle.py | 2 - scrapy/pipelines/files.py | 6 +- scrapy/pipelines/media.py | 2 - scrapy/pqueues.py | 1 - scrapy/shell.py | 2 - scrapy/spidermiddlewares/depth.py | 1 - scrapy/spidermiddlewares/httperror.py | 6 +- scrapy/spidermiddlewares/urllength.py | 1 - scrapy/utils/log.py | 1 - tests/test_crawler.py | 25 ++++++++ 32 files changed, 131 insertions(+), 100 deletions(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 7ff8d3464..598edfeb5 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -35,6 +35,13 @@ how you :ref:`configure the downloader middlewares :class:`scrapy.Spider` subclass and a :class:`scrapy.settings.Settings` object. + The :attr:`engine`, :attr:`extensions`, :attr:`logformatter`, + :attr:`request_fingerprinter` and :attr:`stats` attributes get their value + when the crawl starts, and raise :exc:`RuntimeError` when read before that. + + .. versionchanged:: VERSION + Those attributes used to be ``None`` before getting their value. + .. attribute:: request_fingerprinter The request fingerprint builder of this crawler. diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 51caed57f..b6255ec7e 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -281,7 +281,6 @@ class Command(BaseRunSpiderCommand): ) -> list[Any]: items, requests, opts, depth, spider, callback = args if opts.pipelines: - assert self.pcrawler.engine itemproc = self.pcrawler.engine.scraper.itemproc if hasattr(itemproc, "process_item_async"): for item in items: diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b7a1e0fcd..b63d136a4 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -112,7 +112,6 @@ class ExecutionEngine: self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.signals: SignalManager = crawler.signals - assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter self._slot: _Slot | None = None self.spider: Spider | None = None @@ -299,7 +298,6 @@ class ExecutionEngine: response=None, spider=self.spider, ) - assert self.crawler.stats self.crawler.stats.inc_value("spider_exceptions/count") self.crawler.stats.inc_value( f"spider_exceptions/{type(exception).__name__}" @@ -563,17 +561,17 @@ class ExecutionEngine: if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)): await maybe_deferred_to_future(d) await self.scraper.open_spider_async() - assert self.crawler.stats - if argument_is_required(self.crawler.stats.open_spider, "spider"): + stats = self.crawler.stats + if argument_is_required(stats.open_spider, "spider"): warnings.warn( - f"The open_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument," + f"The open_spider() method of {global_object_name(type(stats))} requires a spider argument," f" this is deprecated and the argument will not be passed in future Scrapy versions.", ScrapyDeprecationWarning, stacklevel=2, ) - self.crawler.stats.open_spider(spider=self.crawler.spider) + stats.open_spider(spider=self.crawler.spider) else: - self.crawler.stats.open_spider() + stats.open_spider() await self.signals.send_catch_log_async( signals.spider_opened, spider=self.crawler.spider ) @@ -676,20 +674,18 @@ class ExecutionEngine: extra={"spider": spider}, ) - assert self.crawler.stats try: - if argument_is_required(self.crawler.stats.close_spider, "spider"): + stats = self.crawler.stats + if argument_is_required(stats.close_spider, "spider"): warnings.warn( - f"The close_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument," + f"The close_spider() method of {global_object_name(type(stats))} requires a spider argument," f" this is deprecated and the argument will not be passed in future Scrapy versions.", ScrapyDeprecationWarning, stacklevel=2, ) - self.crawler.stats.close_spider( - spider=self.crawler.spider, reason=reason - ) + stats.close_spider(spider=self.crawler.spider, reason=reason) else: - self.crawler.stats.close_spider(reason=reason) + stats.close_spider(reason=reason) except Exception: logger.error("Stats close failure") diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 58e37ce5e..6426b1751 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -120,7 +120,6 @@ class Scraper: self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS") self.crawler: Crawler = crawler self.signals: SignalManager = crawler.signals - assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter def _check_deprecated_itemproc_method(self, method: str) -> None: @@ -355,7 +354,6 @@ class Scraper: assert self.crawler.spider exc = _failure.value if isinstance(exc, CloseSpider): - assert self.crawler.engine is not None # typing _schedule_coro( self.crawler.engine.close_spider_async(reason=exc.reason or "cancelled") ) @@ -374,11 +372,9 @@ class Scraper: response=response, spider=self.crawler.spider, ) - assert self.crawler.stats - self.crawler.stats.inc_value("spider_exceptions/count") - self.crawler.stats.inc_value( - f"spider_exceptions/{_failure.value.__class__.__name__}" - ) + stats = self.crawler.stats + stats.inc_value("spider_exceptions/count") + stats.inc_value(f"spider_exceptions/{_failure.value.__class__.__name__}") def handle_spider_output( self, @@ -456,7 +452,6 @@ class Scraper: Items are sent to the item pipelines, requests are scheduled. """ if isinstance(output, Request): - assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) return if output is not None: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e2f726519..44c4ffdcf 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -8,7 +8,7 @@ import signal import warnings from abc import ABC, abstractmethod from functools import partial -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, Generic, TypeVar, overload from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks @@ -58,7 +58,55 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") +class _LateAttribute(Generic[_T]): + """Descriptor for a :class:`Crawler` attribute that only gets a value once + the crawl starts. + + The value is kept in an attribute of the same name prefixed with an + underscore, and reading it before it is set raises :exc:`RuntimeError`. + This way the public attribute can be annotated as always set, and its + users, both in Scrapy and in third-party code, do not need to narrow its + type on every use. Code that runs before the crawl starts reads the + underscore-prefixed attribute instead. + """ + + def __set_name__(self, owner: type[Crawler], name: str) -> None: + self._name = name + self._private_name = f"_{name}" + + @overload + def __get__(self, instance: None, owner: type[Crawler]) -> _LateAttribute[_T]: ... + + @overload + def __get__(self, instance: Crawler, owner: type[Crawler]) -> _T: ... + + def __get__( + self, instance: Crawler | None, owner: type[Crawler] + ) -> _LateAttribute[_T] | _T: + if instance is None: + return self + value: _T | None = getattr(instance, self._private_name) + if value is None: + raise RuntimeError( + f"Crawler.{self._name} is not set yet. It is set when the " + "crawl starts, so it can only be used from then on, e.g. " + "from the spider_opened signal handler onwards." + ) + return value + + def __set__(self, instance: Crawler, value: _T) -> None: + setattr(instance, self._private_name, value) + + class Crawler: + engine: _LateAttribute[ExecutionEngine] = _LateAttribute() + extensions: _LateAttribute[ExtensionManager] = _LateAttribute() + logformatter: _LateAttribute[LogFormatter] = _LateAttribute() + request_fingerprinter: _LateAttribute[RequestFingerprinterProtocol] = ( + _LateAttribute() + ) + stats: _LateAttribute[StatsCollector] = _LateAttribute() + def __init__( self, spidercls: type[Spider], @@ -83,12 +131,13 @@ class Crawler: self.crawling: bool = False self._started: bool = False - self.extensions: ExtensionManager | None = None - self.stats: StatsCollector | None = None - self.logformatter: LogFormatter | None = None - self.request_fingerprinter: RequestFingerprinterProtocol | None = None self.spider: Spider | None = None - self.engine: ExecutionEngine | None = None + + self._engine: ExecutionEngine | None = None + self._extensions: ExtensionManager | None = None + self._logformatter: LogFormatter | None = None + self._request_fingerprinter: RequestFingerprinterProtocol | None = None + self._stats: StatsCollector | None = None def _update_root_log_handler(self) -> None: if get_scrapy_root_handler() is not None: @@ -225,8 +274,8 @@ class Crawler: yield deferred_from_coro(self.engine.start_async()) except Exception: self.crawling = False - if self.engine is not None: - yield deferred_from_coro(self.engine.close_async()) + if self._engine is not None: + yield deferred_from_coro(self._engine.close_async()) raise async def crawl_async(self, *args: Any, **kwargs: Any) -> None: @@ -255,8 +304,8 @@ class Crawler: await self.engine.start_async() except Exception: self.crawling = False - if self.engine is not None: - await self.engine.close_async() + if self._engine is not None: + await self._engine.close_async() raise def _create_spider(self, *args: Any, **kwargs: Any) -> Spider: @@ -282,7 +331,6 @@ class Crawler: """ if self.crawling: self.crawling = False - assert self.engine if self.engine.running: await self.engine.stop_async() @@ -313,7 +361,7 @@ class Crawler: This method can only be called after the crawl engine has been created, e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. """ - if not self.engine: + if self._engine is None: raise RuntimeError( "Crawler.get_downloader_middleware() can only be called after " "the crawl engine has been created." @@ -331,7 +379,7 @@ class Crawler: created, e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. """ - if not self.extensions: + if self._extensions is None: raise RuntimeError( "Crawler.get_extension() can only be called after the " "extension manager has been created." @@ -348,7 +396,7 @@ class Crawler: This method can only be called after the crawl engine has been created, e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. """ - if not self.engine: + if self._engine is None: raise RuntimeError( "Crawler.get_item_pipeline() can only be called after the " "crawl engine has been created." @@ -365,7 +413,7 @@ class Crawler: This method can only be called after the crawl engine has been created, e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. """ - if not self.engine: + if self._engine is None: raise RuntimeError( "Crawler.get_spider_middleware() can only be called after the " "crawl engine has been created." diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index e7ca0ac0e..3ebd98027 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -55,7 +55,6 @@ class HttpCacheMiddleware: @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - assert crawler.stats o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 0d5c9e4c9..b9a26df3a 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -61,7 +61,6 @@ class OffsiteMiddleware: @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index f910d07c8..dd2897ee4 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -94,7 +94,6 @@ def get_retry_request( retry-related job stats """ settings = spider.crawler.settings - assert spider.crawler.stats stats = spider.crawler.stats retry_times = request.meta.get("retry_times", 0) + 1 if max_retry_times is None: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 81a3a887f..d7aa8738c 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -27,6 +27,7 @@ if TYPE_CHECKING: from scrapy import Spider from scrapy.crawler import Crawler from scrapy.robotstxt import RobotParser + from scrapy.statscollectors import StatsCollector logger = logging.getLogger(__name__) @@ -41,6 +42,7 @@ class RobotsTxtMiddleware: self._default_useragent: str = crawler.settings["USER_AGENT"] self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"] self.crawler: Crawler = crawler + self._stats: StatsCollector = crawler.stats self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {} self._parserimpl: RobotParser = load_object( crawler.settings.get("ROBOTSTXT_PARSER") @@ -78,8 +80,7 @@ class RobotsTxtMiddleware: {"request": request}, extra={"spider": self.crawler.spider}, ) - assert self.crawler.stats - self.crawler.stats.inc_value("robotstxt/forbidden") + self._stats.inc_value("robotstxt/forbidden") raise IgnoreRequest("Forbidden by robots.txt") async def robot_parser(self, request: Request) -> RobotParser | None: @@ -95,8 +96,6 @@ class RobotsTxtMiddleware: meta={"dont_obey_robotstxt": True}, callback=NO_CALLBACK, ) - assert self.crawler.engine - assert self.crawler.stats try: resp = await self.crawler.engine.download_async(robotsreq) await self._parse_robots(resp, netloc, request) @@ -109,7 +108,7 @@ class RobotsTxtMiddleware: extra={"spider": self.crawler.spider}, ) self._robots_error(e, netloc) - self.crawler.stats.inc_value("robotstxt/request_count") + self._stats.inc_value("robotstxt/request_count") parser = self._parsers[netloc] if isinstance(parser, Deferred): @@ -119,11 +118,8 @@ class RobotsTxtMiddleware: async def _parse_robots( self, response: Response, netloc: str, request: Request ) -> None: - assert self.crawler.stats - self.crawler.stats.inc_value("robotstxt/response_count") - self.crawler.stats.inc_value( - f"robotstxt/response_status_count/{response.status}" - ) + self._stats.inc_value("robotstxt/response_count") + self._stats.inc_value(f"robotstxt/response_status_count/{response.status}") rp = self._parserimpl.from_crawler(self.crawler, response.body) await self.crawler.signals.send_catch_log_async( signal=signals.robots_parsed, @@ -138,8 +134,7 @@ class RobotsTxtMiddleware: def _robots_error(self, exc: Exception, netloc: str) -> None: if not isinstance(exc, IgnoreRequest): key = f"robotstxt/exception_count/{type(exc)}" - assert self.crawler.stats - self.crawler.stats.inc_value(key) + self._stats.inc_value(key) rp_dfd = self._parsers[netloc] assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = None diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index bafa931de..07de1c2e7 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -43,7 +43,6 @@ class DownloaderStats: def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("DOWNLOADER_STATS"): raise NotConfigured - assert crawler.stats return cls(crawler.stats) @_warn_spider_arg diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 36fb0f97d..09f0be63b 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -95,7 +95,6 @@ class RFPDupeFilter(BaseDupeFilter): @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - assert crawler.request_fingerprinter debug = crawler.settings.getbool("DUPEFILTER_DEBUG") return cls( job_dir(crawler.settings), @@ -134,5 +133,4 @@ class RFPDupeFilter(BaseDupeFilter): self.logger.debug(msg, {"request": request}, extra={"spider": spider}) self.logdupes = False - assert spider.crawler.stats spider.crawler.stats.inc_value("dupefilter/filtered") diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 9cb792e30..5d40e5f9f 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -102,7 +102,6 @@ class CloseSpider: self._close_spider("closespider_pagecount_no_item") def spider_opened(self, spider: Spider) -> None: - assert self.crawler.engine self.task = call_later( self.close_on["timeout"], self._close_spider, "closespider_timeout" ) @@ -146,5 +145,4 @@ class CloseSpider: self._close_spider("closespider_timeout_no_item") def _close_spider(self, reason: str) -> None: - assert self.crawler.engine _schedule_coro(self.crawler.engine.close_spider_async(reason=reason)) diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 6a5e55992..b464942af 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -26,7 +26,6 @@ class CoreStats: @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 5def7509e..8802a3b58 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -45,7 +45,6 @@ class StackTraceDump: return cls(crawler) def dump_stacktrace(self, signum: int, frame: FrameType | None) -> None: - assert self.crawler.engine log_args = { "stackdumps": self._thread_stacks(), "enginestatus": format_engine_status(self.crawler.engine), diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e4294ac90..32abc82d3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -611,7 +611,6 @@ class FeedExporter: logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" slot_type = type(slot.storage).__name__ - assert self.crawler.stats try: await ensure_awaitable(slot.storage.store(self._get_file(slot))) except Exception: diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index dbb79b02d..d008d0f67 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -261,7 +261,6 @@ class DbmCacheStorage: extra={"spider": spider}, ) - assert spider.crawler.request_fingerprinter self._fingerprinter: RequestFingerprinterProtocol = ( spider.crawler.request_fingerprinter ) @@ -326,7 +325,6 @@ class FilesystemCacheStorage: extra={"spider": spider}, ) - assert spider.crawler.request_fingerprinter self._fingerprinter = spider.crawler.request_fingerprinter def close_spider(self, spider: Spider) -> None: diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 6c94d947e..b818569ba 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -37,7 +37,6 @@ class LogStats: interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL") if not interval: raise NotConfigured - assert crawler.stats o = cls(crawler.stats, interval) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 1fde6b296..35ff90e3b 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -29,7 +29,6 @@ class MemoryDebugger: def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("MEMDEBUG_ENABLED"): raise NotConfigured - assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index e0e289ce8..ec761cfbf 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -27,6 +27,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector logger = logging.getLogger(__name__) @@ -43,6 +44,7 @@ class MemoryUsage: raise NotConfigured from exc self.crawler: Crawler = crawler + self._stats: StatsCollector = crawler.stats self.warned: bool = False self.notify_mails: list[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") if self.notify_mails: # pragma: no cover @@ -77,8 +79,7 @@ class MemoryUsage: return size def engine_started(self) -> None: - assert self.crawler.stats - self.crawler.stats.set_value("memusage/startup", self.get_virtual_size()) + self._stats.set_value("memusage/startup", self.get_virtual_size()) self.tasks: list[AsyncioLoopingCall | LoopingCall] = [] tsk = create_looping_call(self.update) self.tasks.append(tsk) @@ -98,15 +99,12 @@ class MemoryUsage: tsk.stop() def update(self) -> None: - assert self.crawler.stats - self.crawler.stats.max_value("memusage/max", self.get_virtual_size()) + self._stats.max_value("memusage/max", self.get_virtual_size()) def _check_limit(self) -> None: - assert self.crawler.engine - assert self.crawler.stats peak_mem_usage = self.get_virtual_size() if peak_mem_usage > self.limit: - self.crawler.stats.set_value("memusage/limit_reached", 1) + self._stats.set_value("memusage/limit_reached", 1) mem = self.limit / 1024 / 1024 logger.error( "Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...", @@ -119,7 +117,7 @@ class MemoryUsage: f"memory usage exceeded {mem}MiB at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) - self.crawler.stats.set_value("memusage/limit_notified", 1) + self._stats.set_value("memusage/limit_notified", 1) if self.crawler.engine.spider is not None: _schedule_coro( @@ -136,9 +134,8 @@ class MemoryUsage: def _check_warning(self) -> None: if self.warned: # warn only once return - assert self.crawler.stats if self.get_virtual_size() > self.warning: - self.crawler.stats.set_value("memusage/warning_reached", 1) + self._stats.set_value("memusage/warning_reached", 1) self.crawler.signals.send_catch_log(signal=signals.memusage_warning_reached) mem = self.warning / 1024 / 1024 logger.warning( @@ -152,16 +149,13 @@ class MemoryUsage: f"memory usage reached {mem}MiB at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) - self.crawler.stats.set_value("memusage/warning_notified", 1) + self._stats.set_value("memusage/warning_notified", 1) self.warned = True def _send_report(self, rcpts: list[str], subject: str) -> None: # pragma: no cover """send notification mail with some additional useful info""" - assert self.crawler.engine - assert self.crawler.stats - stats = self.crawler.stats - s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n" - s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n" + s = f"Memory usage at engine startup : {self._stats.get_value('memusage/startup') / 1024 / 1024}M\r\n" + s += f"Maximum memory usage : {self._stats.get_value('memusage/max') / 1024 / 1024}M\r\n" s += f"Current memory usage : {self.get_virtual_size() / 1024 / 1024}M\r\n" s += ( diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index cbcc8b70e..adffbcbc4 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -87,7 +87,6 @@ class PeriodicLog: ) if not (ext_stats or ext_delta or ext_timing_enabled): raise NotConfigured - assert crawler.stats assert ext_stats is not None assert ext_delta is not None o = cls( diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index f05595806..7647cf33d 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -42,7 +42,6 @@ class StatsMailer: if not recipients: raise NotConfigured mail: MailSender = MailSender.from_crawler(crawler) - assert crawler.stats o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 1506cb1ea..392f79299 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -108,7 +108,6 @@ class TelnetConsole(protocol.ServerFactory): def _get_telnet_vars(self) -> dict[str, Any]: # Note: if you add entries here also update topics/telnetconsole.rst - assert self.crawler.engine telnet_vars: dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index cde73f12e..f44aee03d 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -45,7 +45,6 @@ class AutoThrottle: def _spider_opened(self, spider: Spider) -> None: self.mindelay = self._min_delay() self.maxdelay = self._max_delay() - assert self.crawler.engine self.crawler.engine.downloader._delay = self._start_delay() def _min_delay(self) -> float: @@ -98,7 +97,6 @@ class AutoThrottle: key: str | None = request.meta.get("download_slot") if key is None: return None, None - assert self.crawler.engine return key, self.crawler.engine.downloader.slots.get(key) def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 55a3676e5..e666f4ddd 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -697,9 +697,9 @@ class FilesPipeline(MediaPipeline): } def inc_stats(self, status: str) -> None: - assert self.crawler.stats - self.crawler.stats.inc_value("file_count") - self.crawler.stats.inc_value(f"file_status_count/{status}") + stats = self.crawler.stats + stats.inc_value("file_count") + stats.inc_value(f"file_status_count/{status}") async def _file_downloaded( self, diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 764f82a78..d4025bf3b 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -100,7 +100,6 @@ class MediaPipeline(ABC): stacklevel=2, ) self.crawler: Crawler = crawler - assert crawler.request_fingerprinter self._fingerprinter: RequestFingerprinterProtocol = ( crawler.request_fingerprinter ) @@ -228,7 +227,6 @@ class MediaPipeline(ABC): ) -> FileInfo: try: self._modify_media_request(request) - assert self.crawler.engine response = await self.crawler.engine.download_async(request) return await ensure_awaitable( self.media_downloaded(response, request, info, item=item) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 8e9783f5a..f4c35c0fe 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -265,7 +265,6 @@ class ScrapyPriorityQueue: class DownloaderInterface: def __init__(self, crawler: Crawler): - assert crawler.engine self.downloader: Downloader = crawler.engine.downloader def stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]: diff --git a/scrapy/shell.py b/scrapy/shell.py index dfea00c46..8f78af571 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -193,7 +193,6 @@ class Shell: """ if not self.spider: await self._open_spider(spider) - assert self.crawler.engine is not None # send the request to the engine self.crawler.engine.crawl(request) # this will fire when the request callback runs (via the callback hijacking in _request_deferred()) @@ -204,7 +203,6 @@ class Shell: spider = self.crawler.spider or self.crawler._create_spider() self.crawler.spider = spider - assert self.crawler.engine await self.crawler.engine.open_spider_async(close_if_idle=False) self.spider = spider diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 1781543e3..49683168e 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -70,7 +70,6 @@ class DepthMiddleware(BaseSpiderMiddleware): maxdepth = settings.getint("DEPTH_LIMIT") verbose = settings.getbool("DEPTH_STATS_VERBOSE") prio = settings.getint("DEPTH_PRIORITY") - assert crawler.stats o = cls(maxdepth, crawler.stats, verbose, prio) o.crawler = crawler return o diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 156b73e7e..116a02733 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -78,9 +78,9 @@ class HttpErrorMiddleware: self, response: Response, exception: Exception, spider: Spider | None = None ) -> Iterable[Any] | None: if isinstance(exception, HttpError): - assert self.crawler.stats - self.crawler.stats.inc_value("httperror/response_ignored_count") - self.crawler.stats.inc_value( + stats = self.crawler.stats + stats.inc_value("httperror/response_ignored_count") + stats.inc_value( f"httperror/response_ignored_status_count/{response.status}" ) logger.info( diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index f325ce7a0..86bdd2ed6 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -48,6 +48,5 @@ class UrlLengthMiddleware(BaseSpiderMiddleware): {"maxlength": self.maxlength, "url": request.url}, extra={"spider": self.crawler.spider}, ) - assert self.crawler.stats self.crawler.stats.inc_value("urllength/request_ignored_count") return None diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index ee357c031..bfa39169f 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -239,7 +239,6 @@ class LogCounterHandler(logging.Handler): def emit(self, record: logging.LogRecord) -> None: sname = f"log_count/{record.levelname}" - assert self.crawler.stats self.crawler.stats.inc_value(sname) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 358f20ed7..17ca02dea 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -74,6 +74,31 @@ class TestCrawler: assert not settings.frozen assert crawler.settings.frozen + @pytest.mark.parametrize( + "attr", + ["extensions", "logformatter", "request_fingerprinter", "stats"], + ) + def test_late_attr_before_apply_settings(self, attr: str) -> None: + crawler = get_raw_crawler(DefaultSpider) + with pytest.raises(RuntimeError, match=rf"Crawler\.{attr} is not set yet"): + getattr(crawler, attr) + crawler._apply_settings() + assert getattr(crawler, attr) is not None + + @pytest.mark.parametrize( + "attr", + ["engine", "extensions", "logformatter", "request_fingerprinter", "stats"], + ) + def test_late_attr_on_class(self, attr: str) -> None: + # Introspection tools such as help() read these off the class. + assert getattr(Crawler, attr) is getattr(Crawler, attr) + + def test_late_attr_engine_before_crawl(self) -> None: + crawler = get_raw_crawler(DefaultSpider) + crawler._apply_settings() + with pytest.raises(RuntimeError, match=r"Crawler\.engine is not set yet"): + _ = crawler.engine + @pytest.mark.parametrize( ("attr", "setting"), [ From c285f4cb18c310f131e2ce8912a950d533837180 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 10 Aug 2026 10:06:54 +0200 Subject: [PATCH 101/111] Support CloseSpider during spider startup (#7905) --- docs/topics/item-pipeline.rst | 7 +++++ docs/topics/signals.rst | 7 +++++ scrapy/core/engine.py | 33 +++++++++++++++------- scrapy/crawler.py | 18 ++++++++---- scrapy/exceptions.py | 7 ++--- tests/test_engine.py | 52 ++++++++++++++++++++++++++++++++++- 6 files changed, 104 insertions(+), 20 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 35891ce8e..a6aac78ac 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -47,6 +47,13 @@ Additionally, they may also implement the following methods: This method is called when the spider is opened. + .. versionchanged:: VERSION + Added support for :exc:`~scrapy.exceptions.CloseSpider`. + + It may raise :exc:`~scrapy.exceptions.CloseSpider` to close the spider before + it starts crawling, e.g. if a resource that the pipeline needs is + unavailable. + .. method:: close_spider(self) This method is called when the spider is closed, before the diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index acbf7ddef..0a85c3c05 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -290,6 +290,13 @@ spider_opened reserve per-spider resources, but can be used for any task that needs to be performed when a spider is opened. + .. versionchanged:: VERSION + Added support for :exc:`~scrapy.exceptions.CloseSpider`. + + You may raise a :exc:`~scrapy.exceptions.CloseSpider` exception to close the + spider before it starts crawling, e.g. if a resource that the spider needs + is unavailable. + This signal supports :ref:`asynchronous handlers `. :param spider: the spider which has been opened diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b63d136a4..104daf399 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -248,7 +248,7 @@ class ExecutionEngine: ) return deferred_from_coro(self.close_async()) - async def close_async(self) -> None: + async def close_async(self, *, reason: str = "shutdown") -> None: """ Gracefully close the execution engine. If it has already been started, stop it. In all cases, close the spider and the downloader. @@ -256,9 +256,7 @@ class ExecutionEngine: if self.running: await self.stop_async() # will also close spider and downloader elif self.spider is not None: - await self.close_spider_async( - reason="shutdown" - ) # will also close downloader + await self.close_spider_async(reason=reason) # will also close downloader elif hasattr(self, "downloader"): self.downloader.close() @@ -557,10 +555,20 @@ class ExecutionEngine: nextcall = CallLaterOnce(self._start_scheduled_requests) scheduler = build_from_crawler(self.scheduler_cls, self.crawler) self._slot = _Slot(close_if_idle, nextcall, scheduler) - self._start = await self.scraper.spidermw.process_start() - if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)): - await maybe_deferred_to_future(d) - await self.scraper.open_spider_async() + # A component that fails to start can ask for the spider to be closed. + # The rest of the startup runs anyway, so that components that are + # started also get stopped, and the request is honored once the spider + # is open. + close_spider_exc: CloseSpider | None = None + try: + self._start = await self.scraper.spidermw.process_start() + if hasattr(scheduler, "open") and ( + d := scheduler.open(self.crawler.spider) + ): + await maybe_deferred_to_future(d) + await self.scraper.open_spider_async() + except CloseSpider as exc: + close_spider_exc = exc stats = self.crawler.stats if argument_is_required(stats.open_spider, "spider"): warnings.warn( @@ -572,9 +580,14 @@ class ExecutionEngine: stats.open_spider(spider=self.crawler.spider) else: stats.open_spider() - await self.signals.send_catch_log_async( - signals.spider_opened, spider=self.crawler.spider + results = await self.signals.send_catch_log_async( + signals.spider_opened, spider=self.crawler.spider, dont_log=CloseSpider ) + for _, result in results: + if isinstance(result, CloseSpider): + close_spider_exc = close_spider_exc or result + if close_spider_exc is not None: + raise close_spider_exc def _spider_idle(self) -> None: """ diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 44c4ffdcf..f0cfba6b9 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -15,7 +15,7 @@ from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks from scrapy import Spider from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning from scrapy.extension import ExtensionManager from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings from scrapy.signalmanager import SignalManager @@ -270,8 +270,12 @@ class Crawler: self._apply_settings() self._update_root_log_handler() self.engine = self._create_engine() - yield deferred_from_coro(self.engine.open_spider_async()) - yield deferred_from_coro(self.engine.start_async()) + try: + yield deferred_from_coro(self.engine.open_spider_async()) + except CloseSpider as exc: + yield deferred_from_coro(self.engine.close_async(reason=exc.reason)) + else: + yield deferred_from_coro(self.engine.start_async()) except Exception: self.crawling = False if self._engine is not None: @@ -300,8 +304,12 @@ class Crawler: self._apply_settings() self._update_root_log_handler() self.engine = self._create_engine() - await self.engine.open_spider_async() - await self.engine.start_async() + try: + await self.engine.open_spider_async() + except CloseSpider as exc: + await self.engine.close_async(reason=exc.reason) + else: + await self.engine.start_async() except Exception: self.crawling = False if self._engine is not None: diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index fa5bc1c1e..cd2560df1 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -56,12 +56,11 @@ class DontCloseSpider(Exception): class CloseSpider(Exception): - """Raised from a :ref:`spider callback ` or from - :meth:`~scrapy.Spider.start` to request the spider to be closed/stopped. + """Raised from a :ref:`spider callback `, or while the + spider is starting, to request the spider to be closed/stopped. .. versionchanged:: VERSION - Raising it from :meth:`~scrapy.Spider.start` closes the spider, instead - of being reported as a start error. + Added support for raising it while the spider is starting. *reason* is a string with the reason for closing. diff --git a/tests/test_engine.py b/tests/test_engine.py index 8a3bceccb..53eb4a1f6 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -29,7 +29,9 @@ from tests.utils.engine import ( ) if TYPE_CHECKING: - from collections.abc import AsyncIterator + from collections.abc import AsyncIterator, Generator + + from twisted.internet.defer import Deferred from tests.mockserver.http import MockServer @@ -230,3 +232,51 @@ async def test_request_scheduled_signal(): f"{scheduler.enqueued!r} != [{keep_request!r}]" ) crawler.signals.disconnect(signal_handler, signals.request_scheduled) + + +class ClosingPipeline: + def open_spider(self): + raise CloseSpider("pipeline_reason") + + +class TestCloseSpiderOnStartup: + @coroutine_test + async def test_pipeline(self, caplog: pytest.LogCaptureFixture) -> None: + closed: list[str] = [] + + def spider_closed(reason: str) -> None: + closed.append(reason) + + crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}}) + crawler.signals.connect(spider_closed, signals.spider_closed) + with caplog.at_level(logging.INFO): + await crawler.crawl_async() + assert crawler.stats.get_value("finish_reason") == "pipeline_reason" + assert closed == ["pipeline_reason"] + assert "Traceback" not in caplog.text + + @coroutine_test + async def test_spider_opened(self) -> None: + def spider_opened(spider: Spider) -> None: + raise CloseSpider("signal_reason") + + crawler = get_crawler(DefaultSpider) + crawler.signals.connect(spider_opened, signals.spider_opened) + await crawler.crawl_async() + assert crawler.stats.get_value("finish_reason") == "signal_reason" + + @coroutine_test + async def test_startup_wins_over_spider_opened(self) -> None: + def spider_opened(spider: Spider) -> None: + raise CloseSpider("signal_reason") + + crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}}) + crawler.signals.connect(spider_opened, signals.spider_opened) + await crawler.crawl_async() + assert crawler.stats.get_value("finish_reason") == "pipeline_reason" + + @inline_callbacks_test + def test_deferred_crawl(self) -> Generator[Deferred[Any], Any, None]: + crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}}) + yield crawler.crawl() + assert crawler.stats.get_value("finish_reason") == "pipeline_reason" From afbd2c9320aaac56a5a58fa6157f89fa44cdcef1 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 10 Aug 2026 10:09:57 +0200 Subject: [PATCH 102/111] Let trackref release classes defined at run time (#7922) --- scrapy/utils/trackref.py | 11 +++++++---- tests/test_utils_trackref.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 0bbe68def..2882b7f96 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -14,7 +14,6 @@ This library has a minimal performance impact. from __future__ import annotations -from collections import defaultdict from operator import itemgetter from time import monotonic_ns from types import NoneType @@ -28,8 +27,8 @@ if TYPE_CHECKING: from typing_extensions import Self -live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict( - WeakKeyDictionary +live_refs: WeakKeyDictionary[type, WeakKeyDictionary[object, float]] = ( + WeakKeyDictionary() ) @@ -41,7 +40,11 @@ class object_ref: def __new__(cls, *args: Any, **kwargs: Any) -> Self: obj = object.__new__(cls) - live_refs[cls][obj] = monotonic_ns() + try: + refs = live_refs[cls] + except KeyError: + refs = live_refs[cls] = WeakKeyDictionary() + refs[obj] = monotonic_ns() return obj diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 5458aa603..9585c6d83 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -124,3 +124,13 @@ def test_iter_all(): o2 = Bar() # noqa: F841 o3 = Foo() assert set(trackref.iter_all("Foo")) == {o1, o3} + + +def test_run_time_classes() -> None: + for _ in range(10): + base = type("Baz", (trackref.object_ref,), {}) + base() + del base + garbage_collect() + assert not list(trackref.iter_all("Baz")) + assert sum(1 for cls in trackref.live_refs if cls.__name__ == "Baz") == 0 From c7ee394442da1b30d422dce8d40a849a56e65ff1 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 10 Aug 2026 10:46:20 +0200 Subject: [PATCH 103/111] Address warnings from the VCS workflow (#7966) --- pyproject.toml | 3 +++ tests/ignores.txt | 1 + tests/test_utils_console.py | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 5ec1d05a4..0bdcf6b51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -319,6 +319,9 @@ markers = [ ] filterwarnings = [ "ignore::DeprecationWarning:twisted.web.static", + # Jobs that do not report coverage disable it with --no-cov, which pytest-cov + # warns about because the coverage options below stay in place. + "ignore::pytest_cov.CovDisabledWarning", # Twisted doesn't close failed sockets after CannotListenError: https://github.com/twisted/twisted/issues/6108 "ignore:Exception ignored in. Date: Mon, 10 Aug 2026 10:59:44 +0200 Subject: [PATCH 104/111] Implement browser-like bad header handling for the default download handler (#7806) * Implement browser-like bad header handling for the default download handler * Remove dead code --- docs/topics/download-handlers.rst | 35 ++++++--- scrapy/core/downloader/handlers/http11.py | 86 ++++++++++++++++++++- tests/mockserver/http.py | 2 + tests/mockserver/http_resources.py | 33 ++++++++ tests/test_downloader_handler_httpx.py | 2 + tests/utils/bases/download_handlers_http.py | 39 +++++++++- 6 files changed, 183 insertions(+), 14 deletions(-) diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 94e75ab6f..433a6d139 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -130,17 +130,23 @@ using different handlers. Here is a comparison of some features of the built-in HTTP handlers, see the individual handler docs for more differences: -================== ================= ===================== ==================== -Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler -================== ================= ===================== ==================== -Requires asyncio No No Yes -Requires a reactor Yes Yes No -HTTP/1.1 No Yes Yes -HTTP/2 Yes No Yes -TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl`` -HTTP proxies No Yes Yes -SOCKS proxies No No Yes -================== ================= ===================== ==================== +=================== ================= ===================== ==================== +Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler +=================== ================= ===================== ==================== +Requires asyncio No No Yes +Requires a reactor Yes Yes No +HTTP/1.1 No Yes Yes +HTTP/2 Yes No Yes +TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl`` +HTTP proxies No Yes Yes +SOCKS proxies No No Yes +Bad header handling Not applicable Skip bad Fail +=================== ================= ===================== ==================== + +Bad header handling is what a handler does when a response has a bad header +line, e.g. one with no colon in it, which some servers send. Handlers that skip +bad header lines, like web browsers do, still parse the header lines that follow +them; other handlers also lose those, or cannot download such responses at all. You can find additional HTTP download handlers in the scrapy-download-handlers-incubator_ package. This package is made by the Scrapy @@ -191,6 +197,7 @@ Features and limitations HTTP proxies No (not implemented) SOCKS proxies No (not supported by the library) HTTP/2 Yes +Bad header handling Not applicable (HTTP/2 only) ``response.certificate`` :class:`twisted.internet.ssl.Certificate` object Per-request ``bindaddress`` Yes TLS implementation ``pyOpenSSL``/``cryptography`` @@ -239,11 +246,16 @@ Features and limitations HTTP proxies Yes SOCKS proxies No (not supported by the library) HTTP/2 No (implemented as a separate handler) +Bad header handling Skip bad, like web browsers do ``response.certificate`` :class:`twisted.internet.ssl.Certificate` object Per-request ``bindaddress`` Yes TLS implementation ``pyOpenSSL``/``cryptography`` =========================== ================================================ +.. versionchanged:: VERSION + Bad header lines with no colon in them are now skipped, instead of making + the whole response impossible to download. + Other limitations: - IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` @@ -297,6 +309,7 @@ Features and limitations HTTP proxies Yes SOCKS proxies Yes (SOCKS5) HTTP/2 Yes +Bad header handling Fail (not supported by the library) ``response.certificate`` DER bytes Per-request ``bindaddress`` No (not supported by the library) TLS implementation Standard library ``ssl`` diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 3b5c08666..c288ae792 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -17,12 +17,19 @@ from twisted.internet.defer import Deferred, succeed from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.python.failure import Failure +from twisted.web._newclient import ( + HEADER, + STATUS, + HTTP11ClientProtocol, + HTTPClientParser, +) from twisted.web.client import ( URI, Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, + _HTTP11ClientFactory, ) from twisted.web.client import Response as TxResponse from twisted.web.http import PotentialDataLoss, _DataLoss @@ -60,7 +67,8 @@ from ._base_http import BaseHttpDownloadHandler if TYPE_CHECKING: from twisted.internet.base import ReactorBase - from twisted.internet.interfaces import IConsumer + from twisted.internet.interfaces import IAddress, IConsumer + from twisted.web._newclient import Request as TxRequest # typing.NotRequired requires Python 3.11 from typing_extensions import NotRequired @@ -95,7 +103,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): self._pool.maxPersistentPerHost = crawler.settings.getint( "CONCURRENT_REQUESTS_PER_DOMAIN" ) - self._pool._factory.noisy = False + self._pool._factory = _LenientHTTP11ClientFactory self._contextFactory: IPolicyForHTTPS = _load_context_factory_from_settings( crawler @@ -740,3 +748,77 @@ class _ResponseReader(Protocol): reason = Failure(exc) self._finished.errback(reason) + + +class _LenientHTTPClientParser(HTTPClientParser): + """Response parser that skips bad response header lines, those with no + colon in them, instead of failing to parse the whole response. + + Some servers send such lines, and web browsers skip them and keep parsing + the header lines that follow. See + https://github.com/scrapy/scrapy/issues/210. + """ + + def lineReceived(self, line: bytes) -> None: + # A copy of twisted.web._newclient.HTTPParser.lineReceived() where the + # header name and value are only extracted from header lines that have + # a colon. + + # Handle the normal CR LF case. + if line[-1:] == b"\r": + line = line[:-1] + + if self.state == STATUS: + self.statusReceived(line) # type: ignore[no-untyped-call] + self.state = HEADER + return + + # HEADER is the only other state in which lines are received, as the + # parser switches to raw mode for the response body. + if not line or line[0] not in b" \t": + if self._partialHeader is not None: + header = b"".join(self._partialHeader) + if b":" in header: + name, value = header.split(b":", 1) + self.headerReceived(name, value.strip()) # type: ignore[no-untyped-call] + else: + logger.debug( + f"Skipping the bad response header line {header!r}, as " + f"it has no colon." + ) + if not line: + # Empty line means the header section is over. + self.allHeadersReceived() # type: ignore[no-untyped-call] + else: + # Line not beginning with LWS is another header. + self._partialHeader = [line] + else: + # A line beginning with LWS is a continuation of a header begun on + # a previous line. + self._partialHeader.append(line) # type: ignore[union-attr] + + +class _LenientHTTP11ClientProtocol(HTTP11ClientProtocol): + """Protocol that parses responses with :class:`_LenientHTTPClientParser`.""" + + def request(self, request: TxRequest) -> Deferred[IResponse]: + d: Deferred[IResponse] = super().request(request) + # HTTP11ClientProtocol.request() hardcodes the parser class, so the + # only way to use a different one is to replace the class of the parser + # object that it creates. This is safe because + # _LenientHTTPClientParser defines no additional state. The parser is + # always there because HTTPConnectionPool only reuses connections whose + # protocol is in the QUIESCENT state, for which request() always + # creates a parser. + assert self._parser is not None + self._parser.__class__ = _LenientHTTPClientParser + return d + + +class _LenientHTTP11ClientFactory(_HTTP11ClientFactory): + """Factory that builds :class:`_LenientHTTP11ClientProtocol` protocols.""" + + noisy = False + + def buildProtocol(self, addr: IAddress | None) -> HTTP11ClientProtocol: + return _LenientHTTP11ClientProtocol(self._quiescentCallback) # type: ignore[no-untyped-call] diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py index c4fd4464e..6074f1475 100644 --- a/tests/mockserver/http.py +++ b/tests/mockserver/http.py @@ -11,6 +11,7 @@ from tests import tests_datadir from .http_base import BaseMockServer, main_factory from .http_resources import ( ArbitraryLengthPayloadResource, + BadHeader, BaseResource, BrokenChunkedResource, BrokenDownloadResource, @@ -52,6 +53,7 @@ class Root(BaseResource): put_child(self, b"partial", Partial()) put_child(self, b"drop", Drop()) put_child(self, b"raw", Raw()) + put_child(self, b"bad-header", BadHeader()) put_child(self, b"echo", Echo()) put_child(self, b"payload", PayloadResource()) put_child(self, b"alpayload", ArbitraryLengthPayloadResource()) diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py index cb028bc10..969e29a8c 100644 --- a/tests/mockserver/http_resources.py +++ b/tests/mockserver/http_resources.py @@ -210,6 +210,39 @@ class Raw(LeafResource): request.finish() +class BadHeader(LeafResource): + """Sends a response with a bad header line, one with no colon in it, like + some servers do, between two good ones. + + One of the good header lines is split into two lines, so that handling of + such headers is also covered. + """ + + response = ( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: 5\r\n" + b"Content-Type: text/html\r\n" + b"X-Folded-Header: one\r\n" + b"\ttwo\r\n" + b'\r\n' + b"X-After-Bad-Header: works\r\n" + b"\r\n" + b"Works" + ) + + def render_GET(self, request: Request) -> int: + request.startedWriting = 1 + self.deferRequest(request, 0, self._delayedRender, request) + return NOT_DONE_YET + + def _delayedRender(self, request: Request) -> None: + request.write(self.response) + # Clients that stop parsing headers at the bad one don't get + # Content-Length, so they need the connection to be closed to know that + # the response body is over. + close_connection(request) + + class Echo(LeafResource): def render_GET(self, request: Request) -> bytes: assert request.content diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index 976daacaf..27a44227d 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -61,6 +61,7 @@ class HttpxDownloadHandlerMixin: class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase): handler_supports_bindaddress_meta = False + handler_bad_header_handling = "fail" @pytest.mark.skipif( sys.platform == "darwin", @@ -82,6 +83,7 @@ class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase): class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase): handler_supports_bindaddress_meta = False + handler_bad_header_handling = "fail" tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher" @pytest.mark.skip(reason="The check is Twisted-specific") diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py index 65244b938..9b4a38724 100644 --- a/tests/utils/bases/download_handlers_http.py +++ b/tests/utils/bases/download_handlers_http.py @@ -11,7 +11,7 @@ from contextlib import asynccontextmanager from http import HTTPStatus from ipaddress import IPv4Address from socket import gethostbyname -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar, Literal from urllib.parse import urlparse import pytest @@ -60,6 +60,9 @@ if TYPE_CHECKING: from tests.mockserver.http import MockServer +BadHeaderHandling = Literal["skip-bad", "skip-rest", "fail"] + + class TestHttpBase(ABC): is_secure: bool = False http2: bool = False @@ -72,6 +75,14 @@ class TestHttpBase(ABC): # h2.connection.H2Connection.receive_data()), thus closing all streams that # were using it, and we handle this as a normal exception. handler_supports_http2_dataloss: bool = True + # What the handler does with a bad response header line, e.g. one with no + # colon in it: + # "skip-bad": the bad line is skipped and the header lines that follow it + # are still parsed, which is what web browsers do; + # "skip-rest": the bad line is skipped along with the header lines that + # follow it; + # "fail": the response cannot be downloaded at all. + handler_bad_header_handling: BadHeaderHandling = "skip-bad" # default headers added by the underlying library that cannot be suppressed always_present_req_headers: ClassVar[frozenset[str]] = frozenset() default_handler_settings: ClassVar[dict[str, Any]] = {} @@ -645,6 +656,32 @@ class TestHttpBase(ABC): in caplog.text ) + @coroutine_test + async def test_download_bad_header(self, mockserver: MockServer) -> None: + if self.http2: + pytest.skip("Header lines are specific to HTTP/1.x") + request = Request(mockserver.url("/bad-header", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + if self.handler_bad_header_handling == "fail": + with pytest.raises(DownloadFailedError): + await download_handler.download_request(request) + return + response = await download_handler.download_request(request) + assert response.status == 200 + assert response.body == b"Works" + # the header line that precedes the bad one + assert response.headers.get(b"Content-Type") == b"text/html" + # the header split into two lines, also before the bad one + folded_header = response.headers.get(b"X-Folded-Header") + assert folded_header is not None + # the separator between both parts depends on the handler + assert folded_header.split() == [b"one", b"two"] + # the header line that follows the bad one + expected_value = ( + b"works" if self.handler_bad_header_handling == "skip-bad" else None + ) + assert response.headers.get(b"X-After-Bad-Header") == expected_value + @coroutine_test async def test_download_chunked_content(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/chunked", is_secure=self.is_secure)) From fe96c1f54b8f635e9274a854214b07887b9d8d61 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 10 Aug 2026 11:07:42 +0200 Subject: [PATCH 105/111] Serialize dates and times as ISO 8601 in ScrapyJSONEncoder (#7918) --- docs/topics/extensions.rst | 4 ++-- scrapy/utils/serialize.py | 11 ++--------- tests/test_exporters.py | 4 ++-- tests/test_utils_serialize.py | 10 +++++++++- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 78b38cc3f..6d61cc342 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -374,8 +374,8 @@ This extension periodically logs rich stat data as a JSON object:: "elapsed": 360.008903, "log_interval": 60.0, "log_interval_real": 60.006694, - "start_time": "2023-08-03 23:24:57", - "utcnow": "2023-08-03 23:30:57" + "start_time": "2023-08-03T23:24:57.148903+00:00", + "utcnow": "2023-08-03T23:30:57.157806+00:00" } } diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 5d06bbe30..803b63bef 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -10,18 +10,11 @@ from scrapy.http import Request, Response class ScrapyJSONEncoder(json.JSONEncoder): - DATE_FORMAT = "%Y-%m-%d" - TIME_FORMAT = "%H:%M:%S" - def default(self, o: Any) -> Any: if isinstance(o, set): return list(o) - if isinstance(o, datetime.datetime): - return o.strftime(f"{self.DATE_FORMAT} {self.TIME_FORMAT}") - if isinstance(o, datetime.date): - return o.strftime(self.DATE_FORMAT) - if isinstance(o, datetime.time): - return o.strftime(self.TIME_FORMAT) + if isinstance(o, (datetime.datetime, datetime.date, datetime.time)): + return o.isoformat() if isinstance(o, decimal.Decimal): return str(o) if isinstance(o, defer.Deferred): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 956697a04..4359f4ff4 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -578,7 +578,7 @@ class TestJsonLinesItemExporter(TestBaseItemExporter): self.ie.finish_exporting() del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) - item["time"] = str(item["time"]) + item["time"] = item["time"].isoformat() assert exported == item @@ -661,7 +661,7 @@ class TestJsonItemExporter(TestJsonLinesItemExporter): self.ie.finish_exporting() del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) - item["time"] = str(item["time"]) + item["time"] = item["time"].isoformat() assert exported == [item] diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index 2702c2cce..6becad13e 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -20,11 +20,17 @@ class TestJsonEncoder: def test_encode_decode(self, encoder: ScrapyJSONEncoder) -> None: dt = datetime.datetime(2010, 1, 2, 10, 11, 12) - dts = "2010-01-02 10:11:12" + dts = "2010-01-02T10:11:12" + dt_aware = datetime.datetime( + 2010, 1, 2, 10, 11, 12, 133700, tzinfo=datetime.timezone.utc + ) + dt_awares = "2010-01-02T10:11:12.133700+00:00" d = datetime.date(2010, 1, 2) ds = "2010-01-02" t = datetime.time(10, 11, 12) ts = "10:11:12" + t_us = datetime.time(10, 11, 12, 133700) + t_uss = "10:11:12.133700" dec = Decimal("1000.12") decs = "1000.12" s = {"foo"} @@ -36,7 +42,9 @@ class TestJsonEncoder: ("foo", "foo"), (d, ds), (t, ts), + (t_us, t_uss), (dt, dts), + (dt_aware, dt_awares), (dec, decs), (["foo", d], ["foo", ds]), (s, ss), From f1694269d8c9437461936ed449e81aa2ccb3b7cb Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 10 Aug 2026 11:33:58 +0200 Subject: [PATCH 106/111] Add FTPS support to the FTP feed export storage (#7953) --- docs/topics/feed-exports.rst | 32 +++++++++++++++++++++++++++-- docs/topics/settings.rst | 2 +- scrapy/extensions/feedexport.py | 2 ++ scrapy/settings/default_settings.py | 1 + scrapy/utils/ftp.py | 16 +++++++++++---- tests/keys/__init__.py | 6 +++++- tests/mockserver/ftp.py | 32 +++++++++++++++++++++-------- tests/test_feedexport_storages.py | 19 +++++++++++++++++ 8 files changed, 93 insertions(+), 17 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2f686fd0f..467abc989 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -104,7 +104,8 @@ storage backend types which are defined by the URI scheme. The storages backends supported out of the box are: - :ref:`topics-feed-storage-fs` -- :ref:`topics-feed-storage-ftp` +- :ref:`feed-storage-ftp` +- :ref:`feed-storage-ftps` - :ref:`topics-feed-storage-s3` (requires the :ref:`s3 ` extra) - :ref:`topics-feed-storage-gcs` (requires the :ref:`gcs ` extra) - :ref:`topics-feed-storage-stdout` @@ -168,6 +169,7 @@ you specify a path (e.g. ``/tmp/export.csv``). Alternatively you can also use a :class:`pathlib.Path` object. .. _topics-feed-storage-ftp: +.. _feed-storage-ftp: FTP --- @@ -178,6 +180,9 @@ The feeds are stored in a FTP server. - Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv`` - Required external libraries: none +FTP sends credentials and data in cleartext. Use :ref:`feed-storage-ftps` +instead where possible. + FTP supports two different connection modes: `active or passive `_. Scrapy uses the passive connection mode by default. To use the active connection mode instead, set the @@ -192,6 +197,28 @@ storage backend is: ``True``. This storage backend uses :ref:`delayed file delivery `. +.. _feed-storage-ftps: + +FTPS +---- + +The feeds are stored in a FTP server, over a TLS connection, with the +certificate of the server verified. + +.. versionadded:: VERSION + +- URI scheme: ``ftps`` +- Example URI: ``ftps://user:pass@ftp.example.com/path/to/export.csv`` +- Required external libraries: none + +See :ref:`feed-storage-ftp` for connection modes, the ``overwrite`` default and +file delivery. + +.. note:: For SFTP, an unrelated protocol built on SSH, use + `scrapy-feedexporter-sftp + `_. + + .. _topics-feed-storage-s3: S3 @@ -502,7 +529,7 @@ as a fallback value if that key is not provided for a specific feed definition: - :ref:`topics-feed-storage-fs`: ``False`` - - :ref:`topics-feed-storage-ftp`: ``True`` + - :ref:`feed-storage-ftp` and :ref:`feed-storage-ftps`: ``True`` .. note:: Some FTP servers may not support appending to files (the ``APPE`` FTP command). @@ -624,6 +651,7 @@ Default: "s3": "scrapy.extensions.feedexport.S3FeedStorage", "gs": "scrapy.extensions.feedexport.GCSFeedStorage", "ftp": "scrapy.extensions.feedexport.FTPFeedStorage", + "ftps": "scrapy.extensions.feedexport.FTPFeedStorage", } A dict containing the built-in feed storage backends supported by Scrapy. You diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index de74ad5cf..27ef3f7ef 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1393,7 +1393,7 @@ FEED_TEMPDIR Default: ``None`` The Feed Temp dir allows you to set a custom folder to save crawler -temporary files before uploading with :ref:`FTP feed storage ` and +temporary files before uploading with :ref:`FTP feed storage ` and :ref:`Amazon S3 `. .. setting:: FEED_STORAGE_GCS_ACL diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 32abc82d3..874023e25 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -363,6 +363,7 @@ class FTPFeedStorage(BlockingFeedStorage): self.username: str = u.username or "" self.password: str = unquote(u.password or "") self.path: str = u.path + self.tls: bool = u.scheme == "ftps" self.use_active_mode: bool = use_active_mode self.overwrite: bool = not feed_options or feed_options.get("overwrite", True) @@ -390,6 +391,7 @@ class FTPFeedStorage(BlockingFeedStorage): password=self.password, use_active_mode=self.use_active_mode, overwrite=self.overwrite, + tls=self.tls, ) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a44b36c8a..d7b30b849 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -378,6 +378,7 @@ FEED_STORAGES_BASE = { "": "scrapy.extensions.feedexport.FileFeedStorage", "file": "scrapy.extensions.feedexport.FileFeedStorage", "ftp": "scrapy.extensions.feedexport.FTPFeedStorage", + "ftps": "scrapy.extensions.feedexport.FTPFeedStorage", "gs": "scrapy.extensions.feedexport.GCSFeedStorage", "s3": "scrapy.extensions.feedexport.S3FeedStorage", "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage", diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index a3e7a4306..f08e5303d 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,7 +1,8 @@ import posixpath from contextlib import closing -from ftplib import FTP, error_perm +from ftplib import FTP, FTP_TLS, error_perm from posixpath import dirname +from ssl import create_default_context from typing import IO @@ -29,13 +30,20 @@ def ftp_store_file( password: str, use_active_mode: bool = False, overwrite: bool = True, + tls: bool = False, ) -> None: - """Opens a FTP connection with passed credentials,sets current directory - to the directory extracted from given path, then uploads the file to server + """Opens a FTP connection with passed credentials, sets current directory + to the directory extracted from given path, then uploads the file to server. + + If *tls* is ``True``, the connection is secured with TLS (FTPS), and the + certificate of the server is verified. """ - with FTP() as ftp, closing(file): + ftp = FTP_TLS(context=create_default_context()) if tls else FTP() + with ftp, closing(file): ftp.connect(host, port) ftp.login(username, password) + if isinstance(ftp, FTP_TLS): + ftp.prot_p() if use_active_mode: ftp.set_pasv(False) file.seek(0) diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index 9b73ca4f0..804c9b6a8 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -1,4 +1,5 @@ from datetime import datetime, timedelta, timezone +from ipaddress import IPv4Address from pathlib import Path from cryptography.hazmat.backends import default_backend @@ -12,6 +13,7 @@ from cryptography.hazmat.primitives.serialization import ( from cryptography.x509 import ( CertificateBuilder, DNSName, + IPAddress, Name, NameAttribute, SubjectAlternativeName, @@ -53,7 +55,9 @@ def generate_keys(): .not_valid_before(datetime.now(tz=timezone.utc)) .not_valid_after(datetime.now(tz=timezone.utc) + timedelta(days=10)) .add_extension( - SubjectAlternativeName([DNSName("localhost")]), + SubjectAlternativeName( + [DNSName("localhost"), IPAddress(IPv4Address("127.0.0.1"))] + ), critical=False, ) .sign(key, SHA256(), default_backend()) diff --git a/tests/mockserver/ftp.py b/tests/mockserver/ftp.py index 72760b5ac..d88e953a7 100644 --- a/tests/mockserver/ftp.py +++ b/tests/mockserver/ftp.py @@ -10,7 +10,7 @@ from tempfile import mkdtemp from typing import TYPE_CHECKING from pyftpdlib.authorizers import DummyAuthorizer -from pyftpdlib.handlers import FTPHandler +from pyftpdlib.handlers import FTPHandler, TLS_FTPHandler from pyftpdlib.servers import FTPServer from tests.utils import get_script_run_env @@ -25,28 +25,32 @@ if TYPE_CHECKING: class MockFTPServer: """Creates an FTP server on a random port with a default passwordless user (anonymous) and a temporary root path that you can read from the - :attr:`path` attribute.""" + :attr:`path` attribute. + + If *tls* is ``True``, the server requires FTPS, using the test certificate + from :file:`tests/keys`. + """ proc: Popen[str] port: int path: Path - def __init__(self) -> None: + def __init__(self, tls: bool = False) -> None: self.host: str = "127.0.0.1" + self.tls: bool = tls def __enter__(self) -> Self: self.path = Path(mkdtemp()) self.proc = Popen( - [sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)], + [sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)] + + (["--tls"] if self.tls else []), stderr=PIPE, env=get_script_run_env(), text=True, ) assert self.proc.stderr is not None for line in self.proc.stderr: - if "starting FTP server" in line and ( - m := re.search(r"starting FTP server on ([^ :]+):(\d+),", line) - ): + if m := re.search(r"starting FTPS? .*on ([^ :]+):(\d+),", line): self.port = int(m.group(2)) break else: @@ -68,18 +72,28 @@ class MockFTPServer: self.proc.communicate() def url(self, path: str) -> str: - return f"ftp://{self.host}:{self.port}/{path}" + scheme = "ftps" if self.tls else "ftp" + return f"{scheme}://{self.host}:{self.port}/{path}" def main() -> None: parser = ArgumentParser() parser.add_argument("-d", "--directory", required=True) + parser.add_argument("--tls", action="store_true") args = parser.parse_args() authorizer = DummyAuthorizer() full_permissions = "elradfmwMT" authorizer.add_anonymous(args.directory, perm=full_permissions) - handler = FTPHandler + if args.tls: + keys = Path(__file__).parent.parent / "keys" + handler = TLS_FTPHandler + handler.certfile = str(keys / "localhost.crt") + handler.keyfile = str(keys / "localhost.key") + handler.tls_control_required = True + handler.tls_data_required = True + else: + handler = FTPHandler handler.authorizer = authorizer address = ("127.0.0.1", 0) server = FTPServer(address, handler) diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index d7fb5c9c2..23b39431e 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -7,6 +7,7 @@ import sys import tempfile from io import BytesIO from pathlib import Path +from ssl import SSLCertVerificationError from typing import IO, Any from unittest import mock from urllib.parse import quote @@ -169,6 +170,24 @@ class TestFTPFeedStorage: await self._store(url, b"bar", settings=settings) self._assert_stored(ftp_server.path / filename, b"bar") + @coroutine_test + async def test_tls(self, monkeypatch): + monkeypatch.setenv( + "SSL_CERT_FILE", str(Path(__file__).parent / "keys" / "localhost.crt") + ) + with MockFTPServer(tls=True) as ftp_server: + filename = "file" + await self._store(ftp_server.url(filename), b"foo") + self._assert_stored(ftp_server.path / filename, b"foo") + + @coroutine_test + async def test_tls_untrusted_certificate(self): + with ( + MockFTPServer(tls=True) as ftp_server, + pytest.raises(SSLCertVerificationError), + ): + await self._store(ftp_server.url("file"), b"foo") + def test_uri_auth_quote(self): # RFC3986: 3.2.1. User Information pw_quoted = quote(string.punctuation, safe="") From 15885a8db4683e974fbcbb56a484846108afc350 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 10 Aug 2026 17:55:10 +0200 Subject: [PATCH 107/111] Generalize the use of build_from_crawler() internally (#7808) * Generalize the use of build_from_crawler() internally * Do not use build_from_crawler() for spiders, since they are guaranteed having from_crawler() --- scrapy/core/downloader/__init__.py | 5 +- scrapy/core/scraper.py | 12 ++- scrapy/crawler.py | 4 +- scrapy/downloadermiddlewares/robotstxt.py | 6 +- scrapy/extensions/memusage.py | 3 +- scrapy/extensions/statsmailer.py | 3 +- tests/test_command_shell.py | 2 +- tests/test_downloader_handler_twisted_ftp.py | 2 +- .../test_downloader_handler_twisted_http11.py | 3 +- .../test_downloader_handler_twisted_http2.py | 3 +- tests/test_downloadermiddleware.py | 3 +- tests/test_downloadermiddleware_cookies.py | 21 +++--- ...est_downloadermiddleware_defaultheaders.py | 3 +- ...st_downloadermiddleware_downloadtimeout.py | 3 +- tests/test_downloadermiddleware_httpauth.py | 3 +- tests/test_downloadermiddleware_httpcache.py | 3 +- ...st_downloadermiddleware_httpcompression.py | 31 ++++---- tests/test_downloadermiddleware_httpproxy.py | 3 +- tests/test_downloadermiddleware_offsite.py | 29 ++++---- tests/test_downloadermiddleware_redirect.py | 10 +-- ...wnloadermiddleware_redirect_metarefresh.py | 8 +- tests/test_downloadermiddleware_retry.py | 9 ++- tests/test_downloadermiddleware_robotstxt.py | 43 +++++++---- tests/test_downloadermiddleware_stats.py | 5 +- tests/test_downloadermiddleware_useragent.py | 3 +- tests/test_dupefilters.py | 9 ++- tests/test_engine.py | 3 +- tests/test_extension_debug.py | 11 +-- tests/test_extension_memdebug.py | 7 +- tests/test_extension_memusage.py | 3 +- tests/test_extension_periodic_log.py | 3 +- tests/test_extension_statsmailer.py | 7 +- tests/test_extension_telnet.py | 5 +- tests/test_extension_throttle.py | 18 ++--- tests/test_feedexport.py | 16 ++-- tests/test_feedexport_batch.py | 3 +- tests/test_feedexport_storages.py | 39 ++++++---- tests/test_feedexport_uri_params.py | 15 ++-- tests/test_logformatter.py | 16 ++-- tests/test_logstats.py | 7 +- tests/test_middleware.py | 3 +- tests/test_pipeline_files.py | 63 +++++++++------- tests/test_pipeline_images.py | 56 +++++++------- tests/test_pipeline_media.py | 9 ++- tests/test_pipelines.py | 3 +- tests/test_pqueues.py | 32 ++++---- tests/test_resolver.py | 5 +- tests/test_robotstxt_interface.py | 13 +++- tests/test_scheduler.py | 10 +-- tests/test_spidermiddleware.py | 11 +-- tests/test_spidermiddleware_base.py | 9 ++- tests/test_spidermiddleware_depth.py | 2 +- tests/test_spidermiddleware_httperror.py | 9 ++- tests/test_spidermiddleware_metacopy.py | 7 +- tests/test_spidermiddleware_urllength.py | 2 +- tests/test_spiderstate.py | 3 +- tests/test_squeues_request.py | 21 +++--- tests/test_stats.py | 5 +- tests/utils/bases/redirect.py | 74 +++++++++---------- 59 files changed, 405 insertions(+), 314 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index f9ee62838..2089b1224 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -28,6 +28,7 @@ from scrapy.utils.defer import ( maybe_deferred_to_future, ) from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: from collections.abc import Generator @@ -99,8 +100,8 @@ class Downloader: # AUTOTHROTTLE_START_DELAY. self._delay: float = self.settings.getfloat("DOWNLOAD_DELAY") self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") - self.middleware: DownloaderMiddlewareManager = ( - DownloaderMiddlewareManager.from_crawler(crawler) + self.middleware: DownloaderMiddlewareManager = build_from_crawler( + DownloaderMiddlewareManager, crawler ) self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = None self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict( diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 6426b1751..351375d7b 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -36,7 +36,11 @@ from scrapy.utils.defer import ( ) from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.log import failure_to_exc_info, logformatter_adapter -from scrapy.utils.misc import load_object, warn_on_generator_with_return_value +from scrapy.utils.misc import ( + build_from_crawler, + load_object, + warn_on_generator_with_return_value, +) from scrapy.utils.python import global_object_name from scrapy.utils.spider import iterate_spider_output @@ -102,13 +106,13 @@ class Slot: class Scraper: def __init__(self, crawler: Crawler) -> None: self.slot: Slot | None = None - self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler( - crawler + self.spidermw: SpiderMiddlewareManager = build_from_crawler( + SpiderMiddlewareManager, crawler ) itemproc_cls: type[ItemPipelineManager] = load_object( crawler.settings["ITEM_PROCESSOR"] ) - self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler) + self.itemproc: ItemPipelineManager = build_from_crawler(itemproc_cls, crawler) self._itemproc_has_async: dict[str, bool] = {} for method in [ "open_spider", diff --git a/scrapy/crawler.py b/scrapy/crawler.py index f0cfba6b9..444a5fb67 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -156,7 +156,7 @@ class Crawler: self.stats = load_object(self.settings["STATS_CLASS"])(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) - self.logformatter = lf_cls.from_crawler(self) + self.logformatter = build_from_crawler(lf_cls, self) self.request_fingerprinter = build_from_crawler( load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), @@ -200,7 +200,7 @@ class Crawler: logger.debug("Not using a Twisted reactor") self._apply_reactorless_default_settings() - self.extensions = ExtensionManager.from_crawler(self) + self.extensions = build_from_crawler(ExtensionManager, self) self.settings.freeze() d = dict(overridden_settings(self.settings)) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index d7aa8738c..016cf9acb 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -18,7 +18,7 @@ from scrapy.http.request import NO_CALLBACK from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -49,7 +49,7 @@ class RobotsTxtMiddleware: ) # check if parser dependencies are met, this should throw an error otherwise. - self._parserimpl.from_crawler(self.crawler, b"") + build_from_crawler(self._parserimpl, self.crawler, b"") @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -120,7 +120,7 @@ class RobotsTxtMiddleware: ) -> None: self._stats.inc_value("robotstxt/response_count") self._stats.inc_value(f"robotstxt/response_status_count/{response.status}") - rp = self._parserimpl.from_crawler(self.crawler, response.body) + rp = build_from_crawler(self._parserimpl, self.crawler, response.body) await self.crawler.signals.send_catch_log_async( signal=signals.robots_parsed, robotparser=rp, diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index ec761cfbf..bf1f1bec4 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -19,6 +19,7 @@ from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call from scrapy.utils.defer import _schedule_coro from scrapy.utils.engine import get_engine_status +from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: from twisted.internet.task import LoopingCall @@ -57,7 +58,7 @@ class MemoryUsage: category=ScrapyDeprecationWarning, stacklevel=2, ) - self.mail = MailSender.from_crawler(crawler) + self.mail = build_from_crawler(MailSender, crawler) self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 7647cf33d..3dc38dded 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.mail import MailSender +from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: from twisted.internet.defer import Deferred @@ -41,7 +42,7 @@ class StatsMailer: recipients: list[str] = crawler.settings.getlist("STATSMAILER_RCPTS") if not recipients: raise NotConfigured - mail: MailSender = MailSender.from_crawler(crawler) + mail: MailSender = build_from_crawler(MailSender, crawler) o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 6bc1ebbbb..44178727e 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -365,7 +365,7 @@ class TestShell: crawler.engine = MagicMock() crawler.engine.open_spider_async = AsyncMock() shell = Shell(crawler) - spider = Spider("test") + spider = Spider.from_crawler(crawler, "test") await shell._open_spider(spider) assert shell.spider is spider assert crawler.spider is spider diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py index 14de97b21..28a067fb7 100644 --- a/tests/test_downloader_handler_twisted_ftp.py +++ b/tests/test_downloader_handler_twisted_ftp.py @@ -212,4 +212,4 @@ class TestAnonymousFTP(TestFTPBase): def test_not_configured_without_reactor() -> None: crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) with pytest.raises(NotConfigured): - FTPDownloadHandler.from_crawler(crawler) + build_from_crawler(FTPDownloadHandler, crawler) diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index 32dfd7540..db353750d 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -11,6 +11,7 @@ from scrapy import Spider from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.utils.misc import build_from_crawler from tests.utils.bases.download_handlers_http import ( TestHttpBase, TestHttpProxyBase, @@ -51,7 +52,7 @@ class HTTP11DownloadHandlerMixin: def test_not_configured_without_reactor() -> None: crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) with pytest.raises(NotConfigured): - HTTP11DownloadHandler.from_crawler(crawler) + build_from_crawler(HTTP11DownloadHandler, crawler) class TestHttp(HTTP11DownloadHandlerMixin, TestHttpBase): diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 9d4e161f3..2c3954b5e 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -13,6 +13,7 @@ from scrapy import Spider from scrapy.crawler import Crawler from scrapy.exceptions import DownloadFailedError, NotConfigured from scrapy.http import Request +from scrapy.utils.misc import build_from_crawler from tests.utils.bases.download_handlers_http import ( TestHttpProxyBase, TestHttpsBase, @@ -66,7 +67,7 @@ def test_not_configured_without_reactor() -> None: crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) with pytest.raises(NotConfigured): - H2DownloadHandler.from_crawler(crawler) + build_from_crawler(H2DownloadHandler, crawler) class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 92eb18d33..9b89b0760 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -14,6 +14,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.utils.decorators import coroutine_test @@ -30,7 +31,7 @@ class TestManagerBase: async def get_mwman(self) -> AsyncGenerator[DownloaderMiddlewareManager]: crawler = get_crawler(Spider, self.settings_dict) crawler.spider = crawler._create_spider("foo") - mwman = DownloaderMiddlewareManager.from_crawler(crawler) + mwman = build_from_crawler(DownloaderMiddlewareManager, crawler) crawler.engine = crawler._create_engine() await crawler.engine.open_spider_async() try: diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index e4b66fe10..6c4c80eae 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -10,6 +10,7 @@ from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.http.request import CookiesT, VerboseCookie +from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.request import _to_verbose_cookies from scrapy.utils.spider import DefaultSpider @@ -72,8 +73,8 @@ class TestCookiesMiddleware: def setup_method(self): crawler = get_crawler(DefaultSpider) crawler.spider = crawler._create_spider() - self.mw = CookiesMiddleware.from_crawler(crawler) - self.redirect_middleware = RedirectMiddleware.from_crawler(crawler) + self.mw = build_from_crawler(CookiesMiddleware, crawler) + self.redirect_middleware = build_from_crawler(RedirectMiddleware, crawler) def teardown_method(self): del self.mw @@ -94,19 +95,19 @@ class TestCookiesMiddleware: def test_setting_false_cookies_enabled(self): with pytest.raises(NotConfigured): - CookiesMiddleware.from_crawler( - get_crawler(settings_dict={"COOKIES_ENABLED": False}) + build_from_crawler( + CookiesMiddleware, get_crawler(settings_dict={"COOKIES_ENABLED": False}) ) def test_setting_default_cookies_enabled(self): assert isinstance( - CookiesMiddleware.from_crawler(get_crawler()), CookiesMiddleware + build_from_crawler(CookiesMiddleware, get_crawler()), CookiesMiddleware ) def test_setting_true_cookies_enabled(self): assert isinstance( - CookiesMiddleware.from_crawler( - get_crawler(settings_dict={"COOKIES_ENABLED": True}) + build_from_crawler( + CookiesMiddleware, get_crawler(settings_dict={"COOKIES_ENABLED": True}) ), CookiesMiddleware, ) @@ -115,7 +116,7 @@ class TestCookiesMiddleware: self, caplog: pytest.LogCaptureFixture ) -> None: crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True}) - mw = CookiesMiddleware.from_crawler(crawler) + mw = build_from_crawler(CookiesMiddleware, crawler) caplog.clear() with caplog.at_level( logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies" @@ -145,7 +146,7 @@ class TestCookiesMiddleware: def test_debug_no_cookies(self, caplog: pytest.LogCaptureFixture) -> None: crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True}) - mw = CookiesMiddleware.from_crawler(crawler) + mw = build_from_crawler(CookiesMiddleware, crawler) caplog.clear() with caplog.at_level( logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies" @@ -161,7 +162,7 @@ class TestCookiesMiddleware: self, caplog: pytest.LogCaptureFixture ) -> None: crawler = get_crawler(settings_dict={"COOKIES_DEBUG": False}) - mw = CookiesMiddleware.from_crawler(crawler) + mw = build_from_crawler(CookiesMiddleware, crawler) caplog.clear() with caplog.at_level( logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies" diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 8c89c3ffb..507097df9 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -3,6 +3,7 @@ from __future__ import annotations from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler @@ -13,7 +14,7 @@ def get_defaults_mw() -> tuple[dict[bytes, list[bytes]], DefaultHeadersMiddlewar to_bytes(k): [to_bytes(v)] for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items() } - return defaults, DefaultHeadersMiddleware.from_crawler(crawler) + return defaults, build_from_crawler(DefaultHeadersMiddleware, crawler) def test_process_request(): diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 9b64cf349..7e2d77136 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -5,6 +5,7 @@ from typing import Any from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler @@ -12,7 +13,7 @@ def get_request_spider_mw(settings: dict[str, Any] | None = None): crawler = get_crawler(Spider, settings) spider = crawler._create_spider("foo") request = Request("http://scrapytest.org/") - return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler) + return request, spider, build_from_crawler(DownloadTimeoutMiddleware, crawler) def test_default_download_timeout(): diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index dd5af3bc5..e4b414f45 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -7,6 +7,7 @@ from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler _DOMAIN_NOT_SET = object() @@ -21,7 +22,7 @@ def make_mw( } if domain is not _DOMAIN_NOT_SET: settings["HTTPAUTH_DOMAIN"] = domain - return HttpAuthMiddleware.from_crawler(get_crawler(settings_dict=settings)) + return build_from_crawler(HttpAuthMiddleware, get_crawler(settings_dict=settings)) # --- Spider attribute tests (deprecated) --- diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index dc8228470..50ff5e63f 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -17,6 +17,7 @@ from scrapy.exceptions import IgnoreRequest from scrapy.extensions.httpcache import DummyPolicy from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler if TYPE_CHECKING: @@ -87,7 +88,7 @@ class TestBase: def _middleware(self, **new_settings: Any) -> Generator[HttpCacheMiddleware]: with self._get_crawler(**new_settings) as crawler: assert crawler.spider - mw = HttpCacheMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCacheMiddleware, crawler) mw.spider_opened(crawler.spider) try: yield mw diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a43bb51ba..4e0fd7c8b 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -18,6 +18,7 @@ from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests import tests_datadir @@ -59,7 +60,7 @@ def _skip_if_no_zstd() -> None: class TestHttpCompression: def setup_method(self): self.crawler = get_crawler(Spider) - self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) + self.mw = build_from_crawler(HttpCompressionMiddleware, self.crawler) assert self.crawler.stats self.crawler.stats.open_spider() @@ -93,20 +94,22 @@ class TestHttpCompression: def test_setting_false_compression_enabled(self): with pytest.raises(NotConfigured): - HttpCompressionMiddleware.from_crawler( - get_crawler(settings_dict={"COMPRESSION_ENABLED": False}) + build_from_crawler( + HttpCompressionMiddleware, + get_crawler(settings_dict={"COMPRESSION_ENABLED": False}), ) def test_setting_default_compression_enabled(self): assert isinstance( - HttpCompressionMiddleware.from_crawler(get_crawler()), + build_from_crawler(HttpCompressionMiddleware, get_crawler()), HttpCompressionMiddleware, ) def test_setting_true_compression_enabled(self): assert isinstance( - HttpCompressionMiddleware.from_crawler( - get_crawler(settings_dict={"COMPRESSION_ENABLED": True}) + build_from_crawler( + HttpCompressionMiddleware, + get_crawler(settings_dict={"COMPRESSION_ENABLED": True}), ), HttpCompressionMiddleware, ) @@ -496,7 +499,7 @@ class TestHttpCompression: settings = {"DOWNLOAD_MAXSIZE": 1_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B @@ -525,7 +528,7 @@ class TestHttpCompression: settings = {"DOWNLOAD_MAXSIZE": 1_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse("bomb-gzip") # 11_511_612 B @@ -552,7 +555,7 @@ class TestHttpCompression: crawler = get_crawler(DownloadMaxSizeSpider) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") @@ -584,7 +587,7 @@ class TestHttpCompression: def _test_compression_bomb_request_meta(self, compression_id: str) -> None: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") @@ -616,7 +619,7 @@ class TestHttpCompression: settings = {"DOWNLOAD_WARNSIZE": 10_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") @@ -668,7 +671,7 @@ class TestHttpCompression: crawler = get_crawler(DownloadWarnSizeSpider) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") @@ -721,7 +724,7 @@ class TestHttpCompression: ) -> None: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") response.meta["download_warnsize"] = 10_000_000 @@ -769,7 +772,7 @@ class TestHttpCompression: def _get_truncated_response(self, compression_id: str) -> Response: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(compression_id) truncated_body = response.body[: len(response.body) // 2] diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 54d4601a7..a7a6f2079 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -6,6 +6,7 @@ from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler @@ -20,7 +21,7 @@ class TestHttpProxyMiddleware: def test_not_enabled(self): crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False}) with pytest.raises(NotConfigured): - HttpProxyMiddleware.from_crawler(crawler) + build_from_crawler(HttpProxyMiddleware, crawler) def test_no_environment_proxies(self): os.environ.clear() diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index 1f91dcd4b..40e09be13 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -7,6 +7,7 @@ from scrapy import Request, Spider from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware from scrapy.exceptions import IgnoreRequest from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler UNSET = object() @@ -31,7 +32,7 @@ UNSET = object() def test_process_request_domain_filtering(allowed_domain, url, allowed): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request(url) if allowed: @@ -53,7 +54,7 @@ def test_process_request_domain_filtering(allowed_domain, url, allowed): def test_process_request_dont_filter(value, filtered): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) kwargs: dict[str, Any] = {} if value is not UNSET: @@ -82,7 +83,7 @@ def test_process_request_dont_filter(value, filtered): def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) kwargs: dict[str, Any] = {"meta": {}} if allow_offsite is not UNSET: @@ -111,7 +112,7 @@ def test_process_request_no_allowed_domains(value): if value is not UNSET: kwargs["allowed_domains"] = value crawler.spider = crawler._create_spider(name="a", **kwargs) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request("https://example.com") assert mw.process_request(request) is None @@ -121,7 +122,7 @@ def test_process_request_invalid_domains(): crawler = get_crawler(Spider) allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request("https://a.example") assert mw.process_request(request) is None @@ -150,7 +151,7 @@ def test_process_request_invalid_domains(): def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request(url) if allowed: @@ -172,7 +173,7 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): def test_request_scheduled_dont_filter(value, filtered): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) kwargs: dict[str, Any] = {} if value is not UNSET: @@ -199,7 +200,7 @@ def test_request_scheduled_no_allowed_domains(value): if value is not UNSET: kwargs["allowed_domains"] = value crawler.spider = crawler._create_spider(name="a", **kwargs) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request("https://example.com") mw.request_scheduled(request, crawler.spider) @@ -209,7 +210,7 @@ def test_request_scheduled_invalid_domains(): crawler = get_crawler(Spider) allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request("https://a.example") mw.request_scheduled(request, crawler.spider) @@ -222,7 +223,7 @@ def test_request_scheduled_invalid_domains(): def test_repeated_offsite_domain(): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) req1 = Request("http://other.org/1") req2 = Request("http://other.org/2") @@ -246,7 +247,7 @@ def test_should_follow_override(): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"]) - mw = RootOnlyOffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(RootOnlyOffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) assert mw.process_request(Request("https://example.com/1")) is None with pytest.raises(IgnoreRequest): @@ -256,7 +257,7 @@ def test_should_follow_override(): def test_ignore_request_reason(): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request("http://other.org/1") with pytest.raises( @@ -274,7 +275,7 @@ def test_dynamic_allowed_domains(): crawler = get_crawler(DomainSpider) spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"]) crawler.spider = spider - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(spider) with pytest.raises(IgnoreRequest): @@ -300,7 +301,7 @@ def test_dynamic_allowed_domains_caching(): crawler = get_crawler(DomainSpider) spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"]) crawler.spider = spider - mw = TrackingMiddleware.from_crawler(crawler) + mw = build_from_crawler(TrackingMiddleware, crawler) mw.spider_opened(spider) for _ in range(3): diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index de97aaadb..4cebcb281 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -27,7 +27,7 @@ class TestRedirectMiddleware(TestRedirectBase): def setup_method(self): crawler = get_crawler(DefaultSpider) crawler.spider = crawler._create_spider() - self.mw = self.mwcls.from_crawler(crawler) + self.mw = build_from_crawler(self.mwcls, crawler) def get_response(self, request, location, status=302): headers = {"Location": location} @@ -208,7 +208,7 @@ class TestRedirectMiddleware(TestRedirectBase): response = Response(source_url, headers=resp_headers, status=302) crawler = get_crawler() referer_mw = build_from_crawler(RefererMiddleware, crawler) - redirect_mw = self.mwcls.from_crawler(crawler) + redirect_mw = build_from_crawler(self.mwcls, crawler) redirect_mw._referer_spider_middleware = referer_mw redirect_request = redirect_mw.process_response(source_request, response) if expected_referer: @@ -223,7 +223,7 @@ class TestRedirectMiddleware(TestRedirectBase): source_url, headers={"Referer": "http://example.com/old"} ) response = Response(source_url, headers={"Location": redirect_url}, status=302) - redirect_mw = self.mwcls.from_crawler(get_crawler()) + redirect_mw = build_from_crawler(self.mwcls, get_crawler()) redirect_mw._referer_spider_middleware = None redirect_request = redirect_mw.process_response(source_request, response) assert "Referer" not in redirect_request.headers @@ -352,7 +352,7 @@ class TestRedirectMiddleware(TestRedirectBase): @pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES) def test_redirect_schemes(url, location, target): crawler = get_crawler(Spider) - mw = RedirectMiddleware.from_crawler(crawler) + mw = build_from_crawler(RedirectMiddleware, crawler) request = Request(url) response = Response(url, headers={"Location": location}, status=301) redirect = mw.process_response(request, response) @@ -477,4 +477,4 @@ def test_warning_subclass(caplog): def test_not_configured(): crawler = get_crawler(DefaultSpider, {"REDIRECT_ENABLED": False}) with pytest.raises(NotConfigured): - RedirectMiddleware.from_crawler(crawler) + build_from_crawler(RedirectMiddleware, crawler) diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py index 83dc6825f..3887ddd26 100644 --- a/tests/test_downloadermiddleware_redirect_metarefresh.py +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -32,7 +32,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase): def setup_method(self): crawler = get_crawler(Spider) - self.mw = self.mwcls.from_crawler(crawler) + self.mw = build_from_crawler(self.mwcls, crawler) def _body( self, interval: int = 5, url: str = "http://example.org/newpage" @@ -95,7 +95,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase): """Test that Scrapy 1.x behavior remains possible""" settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]} crawler = get_crawler(Spider, settings) - mw = MetaRefreshMiddleware.from_crawler(crawler) + mw = build_from_crawler(MetaRefreshMiddleware, crawler) req = Request(url="http://example.org") body = ( """