diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 13709efde..ccbf3133b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,6 @@ repos: hooks: - id: sphinx-lint - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.8.3 + rev: 0.8.4 hooks: - id: sphinx-scrapy diff --git a/docs/news.rst b/docs/news.rst index aef567a2a..676e0cbbf 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,16 @@ Release notes ============= +.. _release-2.15.2: + +Scrapy 2.15.2 (2026-04-28) +-------------------------- + +Bug fixes +~~~~~~~~~ + +- Fixed links in https://docs.scrapy.org/llms.txt (:issue:`7467`) + .. _release-2.15.1: Scrapy 2.15.1 (2026-04-23) diff --git a/docs/requirements.in b/docs/requirements.in index 13b9dfb5d..a02e8489b 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -5,4 +5,4 @@ sphinx sphinx-notfound-page sphinx-rtd-theme sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.3 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.4 diff --git a/docs/requirements.txt b/docs/requirements.txt index 4d9eb5454..629da6c4d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0 # via # -r requirements.in # sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@f20366277f2598d0c8a60e55fe282aff2da40dcf +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@eef1f8c3ab3b74b6891752b8f4624373345bae26 # via -r requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy diff --git a/pyproject.toml b/pyproject.toml index 42d00d526..d96790873 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -154,7 +154,7 @@ module = [ ignore_missing_imports = true [tool.bumpversion] -current_version = "2.15.1" +current_version = "2.15.2" commit = true tag = true tag_name = "{new_version}" diff --git a/scrapy/VERSION b/scrapy/VERSION index 3b1fc7950..07d875c2d 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.15.1 +2.15.2 diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 9b4b11f87..7ec86a08b 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -4,7 +4,7 @@ import random from collections import deque from dataclasses import dataclass, field from datetime import datetime -from time import time +from time import monotonic from typing import TYPE_CHECKING, Any from twisted.internet.defer import Deferred, inlineCallbacks @@ -203,7 +203,7 @@ class Downloader: return # Delay queue processing if a download_delay is configured - now = time() + now = monotonic() delay = slot.download_delay() if delay: penalty = delay - now + slot.lastseen @@ -319,7 +319,7 @@ class Downloader: slot.close() def _slot_gc(self, age: float = 60) -> None: - mintime = time() - age + mintime = monotonic() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: self.slots.pop(key).close() diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9d102aff8..439638d33 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -7,7 +7,7 @@ import logging import re from contextlib import suppress from io import BytesIO -from time import time +from time import monotonic from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast from urllib.parse import urldefrag, urlparse @@ -460,7 +460,7 @@ class ScrapyAgent: if isinstance(agent, self._TunnelingAgent): headers.removeHeader(b"Proxy-Authorization") bodyproducer = _RequestBodyProducer(request.body) if request.body else None - start_time = time() + start_time = monotonic() d: Deferred[IResponse] = agent.request( method, to_bytes(url, encoding="ascii"), @@ -489,7 +489,7 @@ class ScrapyAgent: raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.") def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T: - request.meta["download_latency"] = time() - start_time + request.meta["download_latency"] = monotonic() - start_time return result @staticmethod diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index f2774cc07..a4f786363 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,6 +1,6 @@ from __future__ import annotations -from time import time +from time import monotonic from typing import TYPE_CHECKING from urllib.parse import urldefrag @@ -113,7 +113,7 @@ class ScrapyH2Agent: timeout = request.meta.get("download_timeout") or self._connect_timeout agent = self._get_agent(request, timeout) - start_time = time() + start_time = monotonic() d = agent.request(request, spider) d.addCallback(self._cb_latency, request, start_time) @@ -125,7 +125,7 @@ class ScrapyH2Agent: def _cb_latency( response: Response, request: Request, start_time: float ) -> Response: - request.meta["download_latency"] = time() - start_time + request.meta["download_latency"] = monotonic() - start_time return response @staticmethod diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 52d287245..2a550cf78 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -3,7 +3,7 @@ from __future__ import annotations import warnings -from time import time +from time import monotonic, time from typing import TYPE_CHECKING from urllib.parse import urldefrag, urlparse, urlunparse @@ -100,7 +100,9 @@ class ScrapyHTTPClientFactory(ClientFactory): afterFoundGet = False def _build_response(self, body, request): - request.meta["download_latency"] = self.headers_time - self.start_time + request.meta["download_latency"] = ( + self._headers_time_mono - self._start_time_mono + ) status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url, body=body) @@ -153,6 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory): self.response_headers: Headers | None = None self.timeout: float = request.meta.get("download_timeout") or timeout self.start_time: float = time() + self._start_time_mono: float = monotonic() self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback( self._build_response, request ) @@ -200,6 +203,7 @@ class ScrapyHTTPClientFactory(ClientFactory): def gotHeaders(self, headers): self.headers_time = time() + self._headers_time_mono = monotonic() self.response_headers = headers def gotStatus(self, version, status, message): diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 1400f4c9b..6a5e55992 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -5,6 +5,7 @@ Extension for collecting core stats like items scraped and start/finish times from __future__ import annotations from datetime import datetime, timezone +from time import monotonic from typing import TYPE_CHECKING, Any from scrapy import Spider, signals @@ -21,6 +22,7 @@ class CoreStats: def __init__(self, stats: StatsCollector): self.stats: StatsCollector = stats self.start_time: datetime | None = None + self._start_time_mono: float | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -35,13 +37,14 @@ class CoreStats: def spider_opened(self, spider: Spider) -> None: self.start_time = datetime.now(tz=timezone.utc) + self._start_time_mono = monotonic() self.stats.set_value("start_time", self.start_time) def spider_closed(self, spider: Spider, reason: str) -> None: assert self.start_time is not None - finish_time = datetime.now(tz=timezone.utc) - elapsed_time = finish_time - self.start_time - elapsed_time_seconds = elapsed_time.total_seconds() + assert self._start_time_mono is not None + finish_time, finish_time_mono = datetime.now(tz=timezone.utc), monotonic() + elapsed_time_seconds = finish_time_mono - self._start_time_mono self.stats.set_value("elapsed_time_seconds", elapsed_time_seconds) self.stats.set_value("finish_time", finish_time) self.stats.set_value("finish_reason", reason) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index ed4f60785..8f70dbc2e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -167,7 +167,7 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): - self.path: str = file_uri_to_path(uri) if uri.startswith("file://") else uri + self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri feed_options = feed_options or {} self.write_mode: OpenBinaryMode = ( "wb" if feed_options.get("overwrite", False) else "ab" diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 5b0c66658..ecb4c1492 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -172,7 +172,7 @@ class AsyncioLoopingCall: raise ValueError("Interval must be greater than 0") self.interval = interval - self._start_time = time.time() + self._start_time = time.monotonic() if now: self._call() loop = asyncio.get_event_loop() @@ -182,7 +182,7 @@ class AsyncioLoopingCall: """Return the time to sleep until the next call.""" assert self.interval is not None assert self._start_time is not None - now = time.time() + now = time.monotonic() running_for = now - self._start_time return self.interval - (running_for % self.interval) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index fd1ff612c..815f87026 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -4,7 +4,7 @@ import sys from io import StringIO from typing import TYPE_CHECKING from unittest import TestCase -from unittest.mock import Mock, PropertyMock, call, patch +from unittest.mock import MagicMock, Mock, PropertyMock, call, patch from scrapy.commands.check import Command, TextTestResult from tests.test_commands import TestProjectBase @@ -133,7 +133,7 @@ class CheckSpider(scrapy.Spider): def test_printSummary_with_unsuccessful_test_result_without_errors_and_without_failures( self, ) -> None: - result = TextTestResult(Mock(), descriptions=False, verbosity=1) + result = TextTestResult(MagicMock(), descriptions=False, verbosity=1) start_time = 1.0 stop_time = 2.0 result.testsRun = 5 @@ -147,7 +147,7 @@ class CheckSpider(scrapy.Spider): def test_printSummary_with_unsuccessful_test_result_with_only_failures( self, ) -> None: - result = TextTestResult(Mock(), descriptions=False, verbosity=1) + result = TextTestResult(MagicMock(), descriptions=False, verbosity=1) start_time = 1.0 stop_time = 2.0 result.testsRun = 5 @@ -158,7 +158,7 @@ class CheckSpider(scrapy.Spider): mock_write.assert_called_with(" (failures=1)") def test_printSummary_with_unsuccessful_test_result_with_only_errors(self) -> None: - result = TextTestResult(Mock(), descriptions=False, verbosity=1) + result = TextTestResult(MagicMock(), descriptions=False, verbosity=1) start_time = 1.0 stop_time = 2.0 result.testsRun = 5 @@ -171,7 +171,7 @@ class CheckSpider(scrapy.Spider): def test_printSummary_with_unsuccessful_test_result_with_both_failures_and_errors( self, ) -> None: - result = TextTestResult(Mock(), descriptions=False, verbosity=1) + result = TextTestResult(MagicMock(), descriptions=False, verbosity=1) start_time = 1.0 stop_time = 2.0 result.testsRun = 5 diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index e99257b6f..86fc071f8 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json from typing import TYPE_CHECKING, Any import pytest @@ -11,9 +10,13 @@ from twisted.web.http import H2_ENABLED from scrapy import Spider from scrapy.crawler import Crawler -from scrapy.exceptions import NotConfigured, UnsupportedURLSchemeError +from scrapy.exceptions import ( + DownloadFailedError, + NotConfigured, + UnsupportedURLSchemeError, +) from scrapy.http import Request -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import maybe_deferred_to_future from tests.test_downloader_handlers_http_base import ( TestHttpProxyBase, TestHttps11Base, @@ -23,6 +26,7 @@ from tests.test_downloader_handlers_http_base import ( TestHttpsWrongHostnameBase, TestHttpWithCrawlerBase, ) +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from scrapy.core.downloader.handlers import DownloadHandlerProtocol @@ -58,27 +62,16 @@ def test_not_configured_without_reactor() -> None: class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): - HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + http2 = True + handler_supports_http2_dataloss = False - @deferred_f_from_coro_f + @coroutine_test async def test_protocol(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/host", is_secure=self.is_secure)) async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.protocol == "h2" - def test_download_cause_data_loss(self) -> None: # type: ignore[override] - pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_cause_data_loss_double_warning(self) -> None: # type: ignore[override] - pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_allow_data_loss(self) -> None: # type: ignore[override] - pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_allow_data_loss_via_setting(self) -> None: # type: ignore[override] - pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - def test_download_conn_failed(self) -> None: # type: ignore[override] # Unlike HTTP11DownloadHandler which raises it from download_request() # (without any special handling), here ConnectionRefusedError (raised in @@ -87,12 +80,6 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): # DOWNLOAD_TIMEOUT. pytest.skip("The handler doesn't properly reraise ConnectionRefusedError") - def test_download_conn_lost(self) -> None: # type: ignore[override] - pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_conn_aborted(self) -> None: # type: ignore[override] - pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - def test_download_dns_error(self) -> None: # type: ignore[override] # Unlike HTTP11DownloadHandler which raises it from download_request() # (without any special handling), here DNSLookupError (raised in @@ -101,7 +88,7 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): # DOWNLOAD_TIMEOUT. pytest.skip("The handler doesn't properly reraise DNSLookupError") - @deferred_f_from_coro_f + @coroutine_test async def test_concurrent_requests_same_domain( self, mockserver: MockServer ) -> None: @@ -116,7 +103,7 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): assert response2.headers["Content-Length"] == b"79" @pytest.mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") - @deferred_f_from_coro_f + @coroutine_test async def test_connect_request(self, mockserver: MockServer) -> None: request = Request( mockserver.url("/file", is_secure=self.is_secure), method="CONNECT" @@ -125,7 +112,7 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): response = await download_handler.download_request(request) assert response.body == b"" - @deferred_f_from_coro_f + @coroutine_test async def test_custom_content_length_good(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/contentlength", is_secure=self.is_secure)) custom_content_length = str(len(request.body)) @@ -134,7 +121,7 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): response = await download_handler.download_request(request) assert response.text == custom_content_length - @deferred_f_from_coro_f + @coroutine_test async def test_custom_content_length_bad(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/contentlength", is_secure=self.is_secure)) actual_content_length = str(len(request.body)) @@ -154,15 +141,12 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): ) ) - @deferred_f_from_coro_f - async def test_duplicate_header(self, mockserver: MockServer) -> None: - request = Request(mockserver.url("/echo", is_secure=self.is_secure)) - header, value1, value2 = "Custom-Header", "foo", "bar" - request.headers.appendlist(header, value1) - request.headers.appendlist(header, value2) + @coroutine_test + async def test_data_loss_handling(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) async with self.get_dh() as download_handler: - response = await download_handler.download_request(request) - assert json.loads(response.text)["headers"][header] == [value1, value2] + with pytest.raises(DownloadFailedError): + await download_handler.download_request(request) class TestHttps2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): @@ -214,7 +198,7 @@ class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): is_secure = True expected_http_proxy_request_body = b"/" - @deferred_f_from_coro_f + @coroutine_test async def test_download_with_proxy_https_timeout( self, proxy_mockserver: ProxyEchoMockServer ) -> None: @@ -223,7 +207,7 @@ class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): super().test_download_with_proxy_https_timeout(proxy_mockserver) # type: ignore[arg-type] ) - @deferred_f_from_coro_f + @coroutine_test async def test_download_with_proxy_without_http_scheme( self, proxy_mockserver: ProxyEchoMockServer ) -> None: diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 59b6d5754..66edf9325 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -521,7 +521,14 @@ class TestHttpBase(ABC): class TestHttp11Base(TestHttpBase): - """HTTP 1.1 test case""" + http2: bool = False + # RFC 9113 ยง8.1.1 explicitly says that a Content-Length mismatch is a + # stream error (of type PROTOCOL_ERROR) so the client will send + # RST_STREAM. Some libraries do only this while e.g. h2 also closes the + # connection (see handling of ProtocolError in + # h2.connection.H2Connection.receive_data()), thus closing all streams that + # were using it, and we handle this as a normal exception. + handler_supports_http2_dataloss: bool = True @coroutine_test async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None: @@ -638,12 +645,11 @@ class TestHttp11Base(TestHttpBase): response = await download_handler.download_request(request) assert response.body == b"chunked content\n" - @pytest.mark.parametrize("url", ["broken", "broken-chunked"]) @coroutine_test - async def test_download_cause_data_loss( - self, url: str, mockserver: MockServer - ) -> None: - request = Request(mockserver.url(f"/{url}", is_secure=self.is_secure)) + async def test_download_cause_data_loss(self, mockserver: MockServer) -> None: + if self.http2 and not self.handler_supports_http2_dataloss: + pytest.skip("This handler doesn't support dataloss on HTTP/2") + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) async with self.get_dh() as download_handler: with pytest.raises(ResponseDataLossError): await download_handler.download_request(request) @@ -652,6 +658,8 @@ class TestHttp11Base(TestHttpBase): async def test_download_cause_data_loss_double_warning( self, caplog: pytest.LogCaptureFixture, mockserver: MockServer ) -> None: + if self.http2 and not self.handler_supports_http2_dataloss: + pytest.skip("This handler doesn't support dataloss on HTTP/2") request = Request(mockserver.url("/broken", is_secure=self.is_secure)) async with self.get_dh() as download_handler: with pytest.raises(ResponseDataLossError): @@ -663,25 +671,43 @@ class TestHttp11Base(TestHttpBase): # no repeated warning assert "Got data loss" not in caplog.text - @pytest.mark.parametrize("url", ["broken", "broken-chunked"]) @coroutine_test - async def test_download_allow_data_loss( - self, url: str, mockserver: MockServer + async def test_download_allow_data_loss_broken( + self, mockserver: MockServer ) -> None: + if self.http2 and not self.handler_supports_http2_dataloss: + pytest.skip("This handler doesn't support dataloss on HTTP/2") request = Request( - mockserver.url(f"/{url}", is_secure=self.is_secure), + mockserver.url("/broken", is_secure=self.is_secure), meta={"download_fail_on_dataloss": False}, ) async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.flags == ["dataloss"] + assert response.text == "partial" + + @coroutine_test + async def test_download_allow_data_loss_broken_chunked( + self, mockserver: MockServer + ) -> None: + if self.http2: + pytest.skip("Chunked encoding is specific to HTTP/1.1") + request = Request( + mockserver.url("/broken-chunked", is_secure=self.is_secure), + meta={"download_fail_on_dataloss": False}, + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.flags == ["dataloss"] + assert response.text == "chunked content\n" - @pytest.mark.parametrize("url", ["broken", "broken-chunked"]) @coroutine_test async def test_download_allow_data_loss_via_setting( - self, url: str, mockserver: MockServer + self, mockserver: MockServer ) -> None: - request = Request(mockserver.url(f"/{url}", is_secure=self.is_secure)) + if self.http2 and not self.handler_supports_http2_dataloss: + pytest.skip("This handler doesn't support dataloss on HTTP/2") + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) async with self.get_dh( {"DOWNLOAD_FAIL_ON_DATALOSS": False} ) as download_handler: @@ -708,6 +734,12 @@ class TestHttp11Base(TestHttpBase): @coroutine_test async def test_download_conn_aborted(self, mockserver: MockServer) -> None: # copy of TestCrawl.test_retry_conn_aborted() + if self.http2: + # it may be possible to write a separate resource that does something + # suitable on HTTP/2 without sending Content-Length + pytest.skip( + "On HTTP/2 this triggers a Content-Length mismatch error instead." + ) request = Request(mockserver.url("/drop?abort=1", is_secure=self.is_secure)) async with self.get_dh() as download_handler: with pytest.raises(DownloadFailedError): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index dcf412c0f..cbf568524 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -44,7 +44,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterable -def path_to_url(path: Path) -> str: +def path_to_url(path: str | Path) -> str: return urljoin("file:", pathname2url(str(path))) @@ -1293,7 +1293,7 @@ class TestFeedExporterSignals: with tempfile.NamedTemporaryFile(suffix="json") as tmp: settings = { "FEEDS": { - f"file:///{tmp.name}": { + printf_escape(path_to_url(tmp.name)): { "format": "json", }, }, diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 76f8c2512..dbe8d85a5 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -115,8 +115,8 @@ class TestFilesPipeline: req1 = Request("http://foo.bar/baz.txt?fizz") assert file_path(req1) == "full/a2b4913a62f65445aeae2bac08cd8c3b41d7195e.txt" - req2 = Request("http://foo.bar/get_img.php?file=photo.jpg") - assert file_path(req2) == "full/118230fd648f1080c81c234d5e2463ea496f8c05.jpg" + req2 = Request("http://foo.bar/get_img.foo?file=photo.jpg") + assert file_path(req2) == "full/7fc9461c9fd836515bea6983373097203a7d748e.jpg" def test_file_path(self): file_path = self.pipeline.file_path @@ -141,10 +141,10 @@ class TestFilesPipeline: assert ( file_path( Request( - "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg" + "http://www.dfsonline.co.uk/get_prod_image?img=status_0907_mdm.jpg" ) ) - == "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg" + == "full/c67f916ff9d542e822dedf38f9fcb146d1faba78.jpg" ) assert ( file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")) @@ -165,10 +165,10 @@ class TestFilesPipeline: assert ( file_path( Request( - "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha" + "http://www.dfsonline.co.uk/get_prod_image?img=status_0907_mdm.jpg.bohaha" ) ) - == "full/76c00cef2ef669ae65052661f68d451162829507" + == "full/e75f2fa260521b56f6b6a867447b8002d00b5841" ) assert ( file_path( diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 8283d3d5d..ddc12766c 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -120,7 +120,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): match=r"Can't (get|pickle) local object|Can't pickle .*: it's not found as", ) as exc_info: q.push(lambda x: x) - if hasattr(sys, "pypy_version_info"): + if sys.version_info >= (3, 14) or hasattr(sys, "pypy_version_info"): assert isinstance(exc_info.value.__context__, pickle.PicklingError) else: assert isinstance(exc_info.value.__context__, AttributeError) diff --git a/tests/test_stats.py b/tests/test_stats.py index 2869d302e..27af18bb1 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -29,6 +29,7 @@ def spider(crawler: Crawler) -> Spider: class TestCoreStatsExtension: + @mock.patch("scrapy.extensions.corestats.monotonic", return_value=0) @mock.patch("scrapy.extensions.corestats.datetime") def test_core_stats_default_stats_collector( self, mock_datetime: mock.Mock, crawler: Crawler, spider: Spider diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py index d11fbc582..7d39a478e 100644 --- a/tests/test_utils_reactor.py +++ b/tests/test_utils_reactor.py @@ -1,5 +1,4 @@ import asyncio -import warnings import pytest @@ -18,17 +17,6 @@ class TestAsyncio: # the result should depend only on the pytest --reactor argument assert is_asyncio_reactor_installed() == (reactor_pytest == "asyncio") - @pytest.mark.requires_reactor # installs a reactor - def test_install_asyncio_reactor(self): - from twisted.internet import reactor as original_reactor - - with warnings.catch_warnings(record=True) as w: - install_reactor(_asyncio_reactor_path) - assert len(w) == 0, [str(warning) for warning in w] - from twisted.internet import reactor # pylint: disable=reimported - - assert original_reactor == reactor - @pytest.mark.requires_reactor # installs a reactor @pytest.mark.only_asyncio @coroutine_test diff --git a/tox.ini b/tox.ini index 5060a043f..d0fd6680d 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] requires = - sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.3 + sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.4 envlist = pre-commit,pylint,typing,py,docs minversion = 1.7.0