mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'origin/master' into faster-shutdown-2
This commit is contained in:
Adrian Chaves
commit
2b397afe52
|
|
@ -27,6 +27,6 @@ repos:
|
|||
hooks:
|
||||
- id: sphinx-lint
|
||||
- repo: https://github.com/scrapy/sphinx-scrapy
|
||||
rev: 0.8.3
|
||||
rev: 0.8.4
|
||||
hooks:
|
||||
- id: sphinx-scrapy
|
||||
|
|
|
|||
|
|
@ -3,6 +3,16 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.15.2:
|
||||
|
||||
Scrapy 2.15.2 (2026-04-28)
|
||||
--------------------------
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- Fixed links in https://docs.scrapy.org/llms.txt (:issue:`7467`)
|
||||
|
||||
.. _release-2.15.1:
|
||||
|
||||
Scrapy 2.15.1 (2026-04-23)
|
||||
|
|
|
|||
|
|
@ -5,4 +5,4 @@ sphinx
|
|||
sphinx-notfound-page
|
||||
sphinx-rtd-theme
|
||||
sphinx-rtd-dark-mode
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.3
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.4
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0
|
|||
# via
|
||||
# -r requirements.in
|
||||
# sphinx-rtd-dark-mode
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@f20366277f2598d0c8a60e55fe282aff2da40dcf
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@eef1f8c3ab3b74b6891752b8f4624373345bae26
|
||||
# via -r requirements.in
|
||||
sphinx-sitemap==2.9.0
|
||||
# via sphinx-scrapy
|
||||
|
|
|
|||
|
|
@ -154,7 +154,7 @@ module = [
|
|||
ignore_missing_imports = true
|
||||
|
||||
[tool.bumpversion]
|
||||
current_version = "2.15.1"
|
||||
current_version = "2.15.2"
|
||||
commit = true
|
||||
tag = true
|
||||
tag_name = "{new_version}"
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.15.1
|
||||
2.15.2
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import random
|
|||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from time import time
|
||||
from time import monotonic
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from twisted.internet.defer import Deferred, inlineCallbacks
|
||||
|
|
@ -203,7 +203,7 @@ class Downloader:
|
|||
return
|
||||
|
||||
# Delay queue processing if a download_delay is configured
|
||||
now = time()
|
||||
now = monotonic()
|
||||
delay = slot.download_delay()
|
||||
if delay:
|
||||
penalty = delay - now + slot.lastseen
|
||||
|
|
@ -319,7 +319,7 @@ class Downloader:
|
|||
slot.close()
|
||||
|
||||
def _slot_gc(self, age: float = 60) -> None:
|
||||
mintime = time() - age
|
||||
mintime = monotonic() - age
|
||||
for key, slot in list(self.slots.items()):
|
||||
if not slot.active and slot.lastseen + slot.delay < mintime:
|
||||
self.slots.pop(key).close()
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import logging
|
|||
import re
|
||||
from contextlib import suppress
|
||||
from io import BytesIO
|
||||
from time import time
|
||||
from time import monotonic
|
||||
from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast
|
||||
from urllib.parse import urldefrag, urlparse
|
||||
|
||||
|
|
@ -460,7 +460,7 @@ class ScrapyAgent:
|
|||
if isinstance(agent, self._TunnelingAgent):
|
||||
headers.removeHeader(b"Proxy-Authorization")
|
||||
bodyproducer = _RequestBodyProducer(request.body) if request.body else None
|
||||
start_time = time()
|
||||
start_time = monotonic()
|
||||
d: Deferred[IResponse] = agent.request(
|
||||
method,
|
||||
to_bytes(url, encoding="ascii"),
|
||||
|
|
@ -489,7 +489,7 @@ class ScrapyAgent:
|
|||
raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
|
||||
|
||||
def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:
|
||||
request.meta["download_latency"] = time() - start_time
|
||||
request.meta["download_latency"] = monotonic() - start_time
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from time import time
|
||||
from time import monotonic
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import urldefrag
|
||||
|
||||
|
|
@ -113,7 +113,7 @@ class ScrapyH2Agent:
|
|||
timeout = request.meta.get("download_timeout") or self._connect_timeout
|
||||
agent = self._get_agent(request, timeout)
|
||||
|
||||
start_time = time()
|
||||
start_time = monotonic()
|
||||
d = agent.request(request, spider)
|
||||
d.addCallback(self._cb_latency, request, start_time)
|
||||
|
||||
|
|
@ -125,7 +125,7 @@ class ScrapyH2Agent:
|
|||
def _cb_latency(
|
||||
response: Response, request: Request, start_time: float
|
||||
) -> Response:
|
||||
request.meta["download_latency"] = time() - start_time
|
||||
request.meta["download_latency"] = monotonic() - start_time
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from time import time
|
||||
from time import monotonic, time
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import urldefrag, urlparse, urlunparse
|
||||
|
||||
|
|
@ -100,7 +100,9 @@ class ScrapyHTTPClientFactory(ClientFactory):
|
|||
afterFoundGet = False
|
||||
|
||||
def _build_response(self, body, request):
|
||||
request.meta["download_latency"] = self.headers_time - self.start_time
|
||||
request.meta["download_latency"] = (
|
||||
self._headers_time_mono - self._start_time_mono
|
||||
)
|
||||
status = int(self.status)
|
||||
headers = Headers(self.response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
|
||||
|
|
@ -153,6 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
|
|||
self.response_headers: Headers | None = None
|
||||
self.timeout: float = request.meta.get("download_timeout") or timeout
|
||||
self.start_time: float = time()
|
||||
self._start_time_mono: float = monotonic()
|
||||
self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback(
|
||||
self._build_response, request
|
||||
)
|
||||
|
|
@ -200,6 +203,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
|
|||
|
||||
def gotHeaders(self, headers):
|
||||
self.headers_time = time()
|
||||
self._headers_time_mono = monotonic()
|
||||
self.response_headers = headers
|
||||
|
||||
def gotStatus(self, version, status, message):
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Extension for collecting core stats like items scraped and start/finish times
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from time import monotonic
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from scrapy import Spider, signals
|
||||
|
|
@ -21,6 +22,7 @@ class CoreStats:
|
|||
def __init__(self, stats: StatsCollector):
|
||||
self.stats: StatsCollector = stats
|
||||
self.start_time: datetime | None = None
|
||||
self._start_time_mono: float | None = None
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
|
|
@ -35,13 +37,14 @@ class CoreStats:
|
|||
|
||||
def spider_opened(self, spider: Spider) -> None:
|
||||
self.start_time = datetime.now(tz=timezone.utc)
|
||||
self._start_time_mono = monotonic()
|
||||
self.stats.set_value("start_time", self.start_time)
|
||||
|
||||
def spider_closed(self, spider: Spider, reason: str) -> None:
|
||||
assert self.start_time is not None
|
||||
finish_time = datetime.now(tz=timezone.utc)
|
||||
elapsed_time = finish_time - self.start_time
|
||||
elapsed_time_seconds = elapsed_time.total_seconds()
|
||||
assert self._start_time_mono is not None
|
||||
finish_time, finish_time_mono = datetime.now(tz=timezone.utc), monotonic()
|
||||
elapsed_time_seconds = finish_time_mono - self._start_time_mono
|
||||
self.stats.set_value("elapsed_time_seconds", elapsed_time_seconds)
|
||||
self.stats.set_value("finish_time", finish_time)
|
||||
self.stats.set_value("finish_reason", reason)
|
||||
|
|
|
|||
|
|
@ -167,7 +167,7 @@ class StdoutFeedStorage:
|
|||
@implementer(IFeedStorage)
|
||||
class FileFeedStorage:
|
||||
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
|
||||
self.path: str = file_uri_to_path(uri) if uri.startswith("file://") else uri
|
||||
self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri
|
||||
feed_options = feed_options or {}
|
||||
self.write_mode: OpenBinaryMode = (
|
||||
"wb" if feed_options.get("overwrite", False) else "ab"
|
||||
|
|
|
|||
|
|
@ -172,7 +172,7 @@ class AsyncioLoopingCall:
|
|||
raise ValueError("Interval must be greater than 0")
|
||||
|
||||
self.interval = interval
|
||||
self._start_time = time.time()
|
||||
self._start_time = time.monotonic()
|
||||
if now:
|
||||
self._call()
|
||||
loop = asyncio.get_event_loop()
|
||||
|
|
@ -182,7 +182,7 @@ class AsyncioLoopingCall:
|
|||
"""Return the time to sleep until the next call."""
|
||||
assert self.interval is not None
|
||||
assert self._start_time is not None
|
||||
now = time.time()
|
||||
now = time.monotonic()
|
||||
running_for = now - self._start_time
|
||||
return self.interval - (running_for % self.interval)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import sys
|
|||
from io import StringIO
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest import TestCase
|
||||
from unittest.mock import Mock, PropertyMock, call, patch
|
||||
from unittest.mock import MagicMock, Mock, PropertyMock, call, patch
|
||||
|
||||
from scrapy.commands.check import Command, TextTestResult
|
||||
from tests.test_commands import TestProjectBase
|
||||
|
|
@ -133,7 +133,7 @@ class CheckSpider(scrapy.Spider):
|
|||
def test_printSummary_with_unsuccessful_test_result_without_errors_and_without_failures(
|
||||
self,
|
||||
) -> None:
|
||||
result = TextTestResult(Mock(), descriptions=False, verbosity=1)
|
||||
result = TextTestResult(MagicMock(), descriptions=False, verbosity=1)
|
||||
start_time = 1.0
|
||||
stop_time = 2.0
|
||||
result.testsRun = 5
|
||||
|
|
@ -147,7 +147,7 @@ class CheckSpider(scrapy.Spider):
|
|||
def test_printSummary_with_unsuccessful_test_result_with_only_failures(
|
||||
self,
|
||||
) -> None:
|
||||
result = TextTestResult(Mock(), descriptions=False, verbosity=1)
|
||||
result = TextTestResult(MagicMock(), descriptions=False, verbosity=1)
|
||||
start_time = 1.0
|
||||
stop_time = 2.0
|
||||
result.testsRun = 5
|
||||
|
|
@ -158,7 +158,7 @@ class CheckSpider(scrapy.Spider):
|
|||
mock_write.assert_called_with(" (failures=1)")
|
||||
|
||||
def test_printSummary_with_unsuccessful_test_result_with_only_errors(self) -> None:
|
||||
result = TextTestResult(Mock(), descriptions=False, verbosity=1)
|
||||
result = TextTestResult(MagicMock(), descriptions=False, verbosity=1)
|
||||
start_time = 1.0
|
||||
stop_time = 2.0
|
||||
result.testsRun = 5
|
||||
|
|
@ -171,7 +171,7 @@ class CheckSpider(scrapy.Spider):
|
|||
def test_printSummary_with_unsuccessful_test_result_with_both_failures_and_errors(
|
||||
self,
|
||||
) -> None:
|
||||
result = TextTestResult(Mock(), descriptions=False, verbosity=1)
|
||||
result = TextTestResult(MagicMock(), descriptions=False, verbosity=1)
|
||||
start_time = 1.0
|
||||
stop_time = 2.0
|
||||
result.testsRun = 5
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
|
@ -11,9 +10,13 @@ from twisted.web.http import H2_ENABLED
|
|||
|
||||
from scrapy import Spider
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.exceptions import NotConfigured, UnsupportedURLSchemeError
|
||||
from scrapy.exceptions import (
|
||||
DownloadFailedError,
|
||||
NotConfigured,
|
||||
UnsupportedURLSchemeError,
|
||||
)
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
from tests.test_downloader_handlers_http_base import (
|
||||
TestHttpProxyBase,
|
||||
TestHttps11Base,
|
||||
|
|
@ -23,6 +26,7 @@ from tests.test_downloader_handlers_http_base import (
|
|||
TestHttpsWrongHostnameBase,
|
||||
TestHttpWithCrawlerBase,
|
||||
)
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.core.downloader.handlers import DownloadHandlerProtocol
|
||||
|
|
@ -58,27 +62,16 @@ def test_not_configured_without_reactor() -> None:
|
|||
|
||||
|
||||
class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
|
||||
HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
|
||||
http2 = True
|
||||
handler_supports_http2_dataloss = False
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
@coroutine_test
|
||||
async def test_protocol(self, mockserver: MockServer) -> None:
|
||||
request = Request(mockserver.url("/host", is_secure=self.is_secure))
|
||||
async with self.get_dh() as download_handler:
|
||||
response = await download_handler.download_request(request)
|
||||
assert response.protocol == "h2"
|
||||
|
||||
def test_download_cause_data_loss(self) -> None: # type: ignore[override]
|
||||
pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)
|
||||
|
||||
def test_download_cause_data_loss_double_warning(self) -> None: # type: ignore[override]
|
||||
pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)
|
||||
|
||||
def test_download_allow_data_loss(self) -> None: # type: ignore[override]
|
||||
pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)
|
||||
|
||||
def test_download_allow_data_loss_via_setting(self) -> None: # type: ignore[override]
|
||||
pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)
|
||||
|
||||
def test_download_conn_failed(self) -> None: # type: ignore[override]
|
||||
# Unlike HTTP11DownloadHandler which raises it from download_request()
|
||||
# (without any special handling), here ConnectionRefusedError (raised in
|
||||
|
|
@ -87,12 +80,6 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
|
|||
# DOWNLOAD_TIMEOUT.
|
||||
pytest.skip("The handler doesn't properly reraise ConnectionRefusedError")
|
||||
|
||||
def test_download_conn_lost(self) -> None: # type: ignore[override]
|
||||
pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)
|
||||
|
||||
def test_download_conn_aborted(self) -> None: # type: ignore[override]
|
||||
pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON)
|
||||
|
||||
def test_download_dns_error(self) -> None: # type: ignore[override]
|
||||
# Unlike HTTP11DownloadHandler which raises it from download_request()
|
||||
# (without any special handling), here DNSLookupError (raised in
|
||||
|
|
@ -101,7 +88,7 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
|
|||
# DOWNLOAD_TIMEOUT.
|
||||
pytest.skip("The handler doesn't properly reraise DNSLookupError")
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
@coroutine_test
|
||||
async def test_concurrent_requests_same_domain(
|
||||
self, mockserver: MockServer
|
||||
) -> None:
|
||||
|
|
@ -116,7 +103,7 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
|
|||
assert response2.headers["Content-Length"] == b"79"
|
||||
|
||||
@pytest.mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247")
|
||||
@deferred_f_from_coro_f
|
||||
@coroutine_test
|
||||
async def test_connect_request(self, mockserver: MockServer) -> None:
|
||||
request = Request(
|
||||
mockserver.url("/file", is_secure=self.is_secure), method="CONNECT"
|
||||
|
|
@ -125,7 +112,7 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
|
|||
response = await download_handler.download_request(request)
|
||||
assert response.body == b""
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
@coroutine_test
|
||||
async def test_custom_content_length_good(self, mockserver: MockServer) -> None:
|
||||
request = Request(mockserver.url("/contentlength", is_secure=self.is_secure))
|
||||
custom_content_length = str(len(request.body))
|
||||
|
|
@ -134,7 +121,7 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
|
|||
response = await download_handler.download_request(request)
|
||||
assert response.text == custom_content_length
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
@coroutine_test
|
||||
async def test_custom_content_length_bad(self, mockserver: MockServer) -> None:
|
||||
request = Request(mockserver.url("/contentlength", is_secure=self.is_secure))
|
||||
actual_content_length = str(len(request.body))
|
||||
|
|
@ -154,15 +141,12 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
|
|||
)
|
||||
)
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
async def test_duplicate_header(self, mockserver: MockServer) -> None:
|
||||
request = Request(mockserver.url("/echo", is_secure=self.is_secure))
|
||||
header, value1, value2 = "Custom-Header", "foo", "bar"
|
||||
request.headers.appendlist(header, value1)
|
||||
request.headers.appendlist(header, value2)
|
||||
@coroutine_test
|
||||
async def test_data_loss_handling(self, mockserver: MockServer) -> None:
|
||||
request = Request(mockserver.url("/broken", is_secure=self.is_secure))
|
||||
async with self.get_dh() as download_handler:
|
||||
response = await download_handler.download_request(request)
|
||||
assert json.loads(response.text)["headers"][header] == [value1, value2]
|
||||
with pytest.raises(DownloadFailedError):
|
||||
await download_handler.download_request(request)
|
||||
|
||||
|
||||
class TestHttps2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase):
|
||||
|
|
@ -214,7 +198,7 @@ class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
|
|||
is_secure = True
|
||||
expected_http_proxy_request_body = b"/"
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
@coroutine_test
|
||||
async def test_download_with_proxy_https_timeout(
|
||||
self, proxy_mockserver: ProxyEchoMockServer
|
||||
) -> None:
|
||||
|
|
@ -223,7 +207,7 @@ class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
|
|||
super().test_download_with_proxy_https_timeout(proxy_mockserver) # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
@coroutine_test
|
||||
async def test_download_with_proxy_without_http_scheme(
|
||||
self, proxy_mockserver: ProxyEchoMockServer
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -521,7 +521,14 @@ class TestHttpBase(ABC):
|
|||
|
||||
|
||||
class TestHttp11Base(TestHttpBase):
|
||||
"""HTTP 1.1 test case"""
|
||||
http2: bool = False
|
||||
# RFC 9113 §8.1.1 explicitly says that a Content-Length mismatch is a
|
||||
# stream error (of type PROTOCOL_ERROR) so the client will send
|
||||
# RST_STREAM. Some libraries do only this while e.g. h2 also closes the
|
||||
# connection (see handling of ProtocolError in
|
||||
# h2.connection.H2Connection.receive_data()), thus closing all streams that
|
||||
# were using it, and we handle this as a normal exception.
|
||||
handler_supports_http2_dataloss: bool = True
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:
|
||||
|
|
@ -638,12 +645,11 @@ class TestHttp11Base(TestHttpBase):
|
|||
response = await download_handler.download_request(request)
|
||||
assert response.body == b"chunked content\n"
|
||||
|
||||
@pytest.mark.parametrize("url", ["broken", "broken-chunked"])
|
||||
@coroutine_test
|
||||
async def test_download_cause_data_loss(
|
||||
self, url: str, mockserver: MockServer
|
||||
) -> None:
|
||||
request = Request(mockserver.url(f"/{url}", is_secure=self.is_secure))
|
||||
async def test_download_cause_data_loss(self, mockserver: MockServer) -> None:
|
||||
if self.http2 and not self.handler_supports_http2_dataloss:
|
||||
pytest.skip("This handler doesn't support dataloss on HTTP/2")
|
||||
request = Request(mockserver.url("/broken", is_secure=self.is_secure))
|
||||
async with self.get_dh() as download_handler:
|
||||
with pytest.raises(ResponseDataLossError):
|
||||
await download_handler.download_request(request)
|
||||
|
|
@ -652,6 +658,8 @@ class TestHttp11Base(TestHttpBase):
|
|||
async def test_download_cause_data_loss_double_warning(
|
||||
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
|
||||
) -> None:
|
||||
if self.http2 and not self.handler_supports_http2_dataloss:
|
||||
pytest.skip("This handler doesn't support dataloss on HTTP/2")
|
||||
request = Request(mockserver.url("/broken", is_secure=self.is_secure))
|
||||
async with self.get_dh() as download_handler:
|
||||
with pytest.raises(ResponseDataLossError):
|
||||
|
|
@ -663,25 +671,43 @@ class TestHttp11Base(TestHttpBase):
|
|||
# no repeated warning
|
||||
assert "Got data loss" not in caplog.text
|
||||
|
||||
@pytest.mark.parametrize("url", ["broken", "broken-chunked"])
|
||||
@coroutine_test
|
||||
async def test_download_allow_data_loss(
|
||||
self, url: str, mockserver: MockServer
|
||||
async def test_download_allow_data_loss_broken(
|
||||
self, mockserver: MockServer
|
||||
) -> None:
|
||||
if self.http2 and not self.handler_supports_http2_dataloss:
|
||||
pytest.skip("This handler doesn't support dataloss on HTTP/2")
|
||||
request = Request(
|
||||
mockserver.url(f"/{url}", is_secure=self.is_secure),
|
||||
mockserver.url("/broken", is_secure=self.is_secure),
|
||||
meta={"download_fail_on_dataloss": False},
|
||||
)
|
||||
async with self.get_dh() as download_handler:
|
||||
response = await download_handler.download_request(request)
|
||||
assert response.flags == ["dataloss"]
|
||||
assert response.text == "partial"
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_allow_data_loss_broken_chunked(
|
||||
self, mockserver: MockServer
|
||||
) -> None:
|
||||
if self.http2:
|
||||
pytest.skip("Chunked encoding is specific to HTTP/1.1")
|
||||
request = Request(
|
||||
mockserver.url("/broken-chunked", is_secure=self.is_secure),
|
||||
meta={"download_fail_on_dataloss": False},
|
||||
)
|
||||
async with self.get_dh() as download_handler:
|
||||
response = await download_handler.download_request(request)
|
||||
assert response.flags == ["dataloss"]
|
||||
assert response.text == "chunked content\n"
|
||||
|
||||
@pytest.mark.parametrize("url", ["broken", "broken-chunked"])
|
||||
@coroutine_test
|
||||
async def test_download_allow_data_loss_via_setting(
|
||||
self, url: str, mockserver: MockServer
|
||||
self, mockserver: MockServer
|
||||
) -> None:
|
||||
request = Request(mockserver.url(f"/{url}", is_secure=self.is_secure))
|
||||
if self.http2 and not self.handler_supports_http2_dataloss:
|
||||
pytest.skip("This handler doesn't support dataloss on HTTP/2")
|
||||
request = Request(mockserver.url("/broken", is_secure=self.is_secure))
|
||||
async with self.get_dh(
|
||||
{"DOWNLOAD_FAIL_ON_DATALOSS": False}
|
||||
) as download_handler:
|
||||
|
|
@ -708,6 +734,12 @@ class TestHttp11Base(TestHttpBase):
|
|||
@coroutine_test
|
||||
async def test_download_conn_aborted(self, mockserver: MockServer) -> None:
|
||||
# copy of TestCrawl.test_retry_conn_aborted()
|
||||
if self.http2:
|
||||
# it may be possible to write a separate resource that does something
|
||||
# suitable on HTTP/2 without sending Content-Length
|
||||
pytest.skip(
|
||||
"On HTTP/2 this triggers a Content-Length mismatch error instead."
|
||||
)
|
||||
request = Request(mockserver.url("/drop?abort=1", is_secure=self.is_secure))
|
||||
async with self.get_dh() as download_handler:
|
||||
with pytest.raises(DownloadFailedError):
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ if TYPE_CHECKING:
|
|||
from collections.abc import Callable, Iterable
|
||||
|
||||
|
||||
def path_to_url(path: Path) -> str:
|
||||
def path_to_url(path: str | Path) -> str:
|
||||
return urljoin("file:", pathname2url(str(path)))
|
||||
|
||||
|
||||
|
|
@ -1293,7 +1293,7 @@ class TestFeedExporterSignals:
|
|||
with tempfile.NamedTemporaryFile(suffix="json") as tmp:
|
||||
settings = {
|
||||
"FEEDS": {
|
||||
f"file:///{tmp.name}": {
|
||||
printf_escape(path_to_url(tmp.name)): {
|
||||
"format": "json",
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -115,8 +115,8 @@ class TestFilesPipeline:
|
|||
req1 = Request("http://foo.bar/baz.txt?fizz")
|
||||
assert file_path(req1) == "full/a2b4913a62f65445aeae2bac08cd8c3b41d7195e.txt"
|
||||
|
||||
req2 = Request("http://foo.bar/get_img.php?file=photo.jpg")
|
||||
assert file_path(req2) == "full/118230fd648f1080c81c234d5e2463ea496f8c05.jpg"
|
||||
req2 = Request("http://foo.bar/get_img.foo?file=photo.jpg")
|
||||
assert file_path(req2) == "full/7fc9461c9fd836515bea6983373097203a7d748e.jpg"
|
||||
|
||||
def test_file_path(self):
|
||||
file_path = self.pipeline.file_path
|
||||
|
|
@ -141,10 +141,10 @@ class TestFilesPipeline:
|
|||
assert (
|
||||
file_path(
|
||||
Request(
|
||||
"http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg"
|
||||
"http://www.dfsonline.co.uk/get_prod_image?img=status_0907_mdm.jpg"
|
||||
)
|
||||
)
|
||||
== "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg"
|
||||
== "full/c67f916ff9d542e822dedf38f9fcb146d1faba78.jpg"
|
||||
)
|
||||
assert (
|
||||
file_path(Request("http://www.dorma.co.uk/images/product_details/2532/"))
|
||||
|
|
@ -165,10 +165,10 @@ class TestFilesPipeline:
|
|||
assert (
|
||||
file_path(
|
||||
Request(
|
||||
"http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha"
|
||||
"http://www.dfsonline.co.uk/get_prod_image?img=status_0907_mdm.jpg.bohaha"
|
||||
)
|
||||
)
|
||||
== "full/76c00cef2ef669ae65052661f68d451162829507"
|
||||
== "full/e75f2fa260521b56f6b6a867447b8002d00b5841"
|
||||
)
|
||||
assert (
|
||||
file_path(
|
||||
|
|
|
|||
|
|
@ -120,7 +120,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
|
|||
match=r"Can't (get|pickle) local object|Can't pickle .*: it's not found as",
|
||||
) as exc_info:
|
||||
q.push(lambda x: x)
|
||||
if hasattr(sys, "pypy_version_info"):
|
||||
if sys.version_info >= (3, 14) or hasattr(sys, "pypy_version_info"):
|
||||
assert isinstance(exc_info.value.__context__, pickle.PicklingError)
|
||||
else:
|
||||
assert isinstance(exc_info.value.__context__, AttributeError)
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ def spider(crawler: Crawler) -> Spider:
|
|||
|
||||
|
||||
class TestCoreStatsExtension:
|
||||
@mock.patch("scrapy.extensions.corestats.monotonic", return_value=0)
|
||||
@mock.patch("scrapy.extensions.corestats.datetime")
|
||||
def test_core_stats_default_stats_collector(
|
||||
self, mock_datetime: mock.Mock, crawler: Crawler, spider: Spider
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import asyncio
|
||||
import warnings
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -18,17 +17,6 @@ class TestAsyncio:
|
|||
# the result should depend only on the pytest --reactor argument
|
||||
assert is_asyncio_reactor_installed() == (reactor_pytest == "asyncio")
|
||||
|
||||
@pytest.mark.requires_reactor # installs a reactor
|
||||
def test_install_asyncio_reactor(self):
|
||||
from twisted.internet import reactor as original_reactor
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
install_reactor(_asyncio_reactor_path)
|
||||
assert len(w) == 0, [str(warning) for warning in w]
|
||||
from twisted.internet import reactor # pylint: disable=reimported
|
||||
|
||||
assert original_reactor == reactor
|
||||
|
||||
@pytest.mark.requires_reactor # installs a reactor
|
||||
@pytest.mark.only_asyncio
|
||||
@coroutine_test
|
||||
|
|
|
|||
Loading…
Reference in New Issue