diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py
index fad7e6f6b..08e8cad24 100644
--- a/scrapy/utils/_deps_compat.py
+++ b/scrapy/utils/_deps_compat.py
@@ -1,7 +1,15 @@
+import sys
+
from OpenSSL import __version__ as PYOPENSSL_VERSION_STRING
from packaging.version import Version
from twisted import version as TWISTED_VERSION
from twisted.python.versions import Version as TxVersion
+from w3lib import __version__ as W3LIB_VERSION_STRING
+
+# improved urllib.robotparser, https://github.com/python/cpython/pull/149374
+STDLIB_IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5) or (
+ (3, 13, 14) <= sys.version_info < (3, 14)
+)
TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0)
# changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506
@@ -14,3 +22,7 @@ PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)
PYOPENSSL_WANTS_X509_PKEY = PYOPENSSL_VERSION < Version("24.3.0")
# SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable
PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0")
+
+W3LIB_VERSION = Version(W3LIB_VERSION_STRING)
+# safe_url_string() strips the input, https://github.com/scrapy/w3lib/pull/207
+W3LIB_STRIPS_URLS = W3LIB_VERSION >= Version("2.1.1")
diff --git a/tests/ignores.txt b/tests/ignores.txt
index 222288841..94edcf186 100644
--- a/tests/ignores.txt
+++ b/tests/ignores.txt
@@ -1,3 +1 @@
-scrapy/downloadermiddlewares/cookies.py
scrapy/extensions/statsmailer.py
-scrapy/extensions/memusage.py
diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.py
index 943775fa5..fdea666e1 100644
--- a/tests/mockserver/simple_https.py
+++ b/tests/mockserver/simple_https.py
@@ -33,7 +33,7 @@ class SimpleMockServer(BaseMockServer):
super().__init__()
self.keyfile = keyfile
self.certfile = certfile
- self.cipher_string = cipher_string or ""
+ self.cipher_string = cipher_string
self.tls_min_version = tls_min_version
self.tls_max_version = tls_max_version
diff --git a/tests/spiders.py b/tests/spiders.py
index 55d1ea365..612dc11c9 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -222,7 +222,7 @@ class AsyncDefDeferredWrappedSpider(SimpleSpider):
class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
- name = "asyncdef_deferred_wrapped"
+ name = "asyncdef_deferred_maybe_wrapped"
async def parse(self, response):
await maybe_deferred_to_future(defer.succeed(None))
diff --git a/tests/test_addons.py b/tests/test_addons.py
index db0fb2f31..14ebddda8 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -161,6 +161,7 @@ class TestAddonManager:
settings.set("KEY", 0, priority="default")
runner = runner_cls(settings)
crawler = runner.create_crawler(Spider)
+ crawler._apply_settings()
assert crawler.settings.getint("KEY") == 20
def test_fallback_workflow(self):
diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py
index 11c821f8d..ef1e50c0c 100644
--- a/tests/test_cmdline/extensions.py
+++ b/tests/test_cmdline/extensions.py
@@ -1,14 +1,2 @@
-"""A test extension used to check the settings loading order"""
-
-
-class TestExtension:
- def __init__(self, settings):
- settings.set("TEST1", f"{settings['TEST1']} + started")
-
- @classmethod
- def from_crawler(cls, crawler):
- return cls(crawler.settings)
-
-
class DummyExtension:
pass
diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py
index 32b15e191..ec71bba0c 100644
--- a/tests/test_cmdline/settings.py
+++ b/tests/test_cmdline/settings.py
@@ -1,7 +1,7 @@
from pathlib import Path
EXTENSIONS = {
- "tests.test_cmdline.extensions.TestExtension": 0,
+ "tests.test_cmdline.extensions.DummyExtension": 0,
}
TEST1 = "default"
diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py
index 2a9d0ed57..7ac6c4fb0 100644
--- a/tests/test_command_startproject.py
+++ b/tests/test_command_startproject.py
@@ -257,7 +257,7 @@ class TestStartprojectTemplates:
assert actual_permissions == expected_permissions
- def test_startproject_permissions_umask_022(self, tmp_path: Path) -> None:
+ def test_startproject_permissions_umask_002(self, tmp_path: Path) -> None:
"""Check that generated files have the right permissions when the
system uses a umask value that causes new files to have different
permissions than those from the template folder."""
diff --git a/tests/test_commands.py b/tests/test_commands.py
index edb03da1b..0657f393c 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -214,9 +214,7 @@ class MySpider(scrapy.Spider):
self._append_settings(proj_path / self.project_name, "TWISTED_REACTOR = None\n")
self._assert_spider_works(self.NORMAL_MSG, proj_path, "sp")
- self._assert_spider_asyncio_fail(
- self.NORMAL_MSG, proj_path, "aiosp", "-s", "TWISTED_REACTOR="
- )
+ self._assert_spider_asyncio_fail(self.NORMAL_MSG, proj_path, "aiosp")
def test_spider_settings_asyncio(self, proj_path: Path) -> None:
"""The reactor is set via the spider settings to the asyncio value.
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 008e326ec..e80945b93 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -388,7 +388,7 @@ class TestContractsManager:
request = self.conman.from_method(spider.returns_item_meta, self.results)
assert request.meta["key"] == "example"
response.meta = request.meta
- request.callback(ResponseMetaMock)
+ request.callback(response)
assert response.meta["key"] == "example"
self.should_succeed()
@@ -476,14 +476,14 @@ class TestContractsManager:
# invalid regex
request = self.conman.from_method(spider.invalid_regex, self.results)
- self.should_succeed()
+ assert request is None
# invalid regex with valid contract
request = self.conman.from_method(
spider.invalid_regex_with_valid_contract, self.results
)
- self.should_succeed()
request.callback(response)
+ self.should_succeed()
def test_custom_contracts(self):
self.conman.from_spider(CustomContractSuccessSpider(), self.results)
@@ -578,7 +578,7 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = CustomFailContractPreProcess(spider.returns_request)
- conman = ContractsManager([contract])
+ conman = ContractsManager([UrlContract, ReturnsContract, contract])
request = conman.from_method(spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
@@ -592,7 +592,7 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = CustomFailContractPostProcess(spider.returns_request)
- conman = ContractsManager([contract])
+ conman = ContractsManager([UrlContract, ReturnsContract, contract])
request = conman.from_method(spider.returns_request, self.results)
contract.add_post_hook(request, self.results)
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index abeaa2f65..e348bfb7f 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -36,7 +36,6 @@ from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
- from twisted.internet.ssl import ContextFactory
from twisted.web.iweb import IBodyProducer
@@ -48,8 +47,6 @@ class TestSlot:
@pytest.mark.requires_reactor # this test is related to the Twisted HTTP code
class TestContextFactoryBase:
- context_factory: ContextFactory | None = None
-
@async_yield_fixture
async def server_url(self, tmp_path):
(tmp_path / "file").write_bytes(b"0123456789")
@@ -69,7 +66,7 @@ class TestContextFactoryBase:
return reactor.listenSSL(
0,
site,
- contextFactory=self.context_factory or ssl_context_factory(),
+ contextFactory=ssl_context_factory(),
interface="127.0.0.1",
)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index ada4c31ce..85d98f847 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -342,7 +342,7 @@ with multiples lines
assert "responses" in crawler.spider.meta
assert "failures" not in crawler.spider.meta
# start() doesn't set Referer header
- echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body))
+ echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][0].body))
assert "Referer" not in echo0["headers"]
# following request sets Referer to the source request url
echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body))
@@ -390,7 +390,7 @@ with multiples lines
est = [x for sublist in est for x in sublist] # flatten
est = [x.lstrip().rstrip() for x in est]
it = iter(est)
- s = dict(zip(it, it, strict=False))
+ s = dict(zip(it, it, strict=True))
assert s["engine.spider.name"] == crawler.spider.name
assert s["len(engine.scraper.slot.active)"] == "1"
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 82956735b..31d195c4b 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -240,11 +240,7 @@ class TestCrawler(TestBaseCrawler):
@classmethod
def from_crawler(cls, crawler):
- try:
- crawler.get_downloader_middleware(DefaultSpider)
- except Exception as e:
- MySpider.result = e
- raise
+ crawler.get_downloader_middleware(DefaultSpider)
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with pytest.raises(RuntimeError):
@@ -322,11 +318,7 @@ class TestCrawler(TestBaseCrawler):
@classmethod
def from_crawler(cls, crawler):
- try:
- crawler.get_extension(DefaultSpider)
- except Exception as e:
- MySpider.result = e
- raise
+ crawler.get_extension(DefaultSpider)
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with pytest.raises(RuntimeError):
@@ -404,11 +396,7 @@ class TestCrawler(TestBaseCrawler):
@classmethod
def from_crawler(cls, crawler):
- try:
- crawler.get_item_pipeline(DefaultSpider)
- except Exception as e:
- MySpider.result = e
- raise
+ crawler.get_item_pipeline(DefaultSpider)
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with pytest.raises(RuntimeError):
@@ -486,11 +474,7 @@ class TestCrawler(TestBaseCrawler):
@classmethod
def from_crawler(cls, crawler):
- try:
- crawler.get_spider_middleware(DefaultSpider)
- except Exception as e:
- MySpider.result = e
- raise
+ crawler.get_spider_middleware(DefaultSpider)
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with pytest.raises(RuntimeError):
@@ -755,11 +739,12 @@ class TestCrawlerRunnerHasSpider:
):
await self._crawl(runner, NoRequestsSpider)
else:
- CrawlerRunner(
+ runner = CrawlerRunner(
settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
)
+ await self._crawl(runner, NoRequestsSpider)
@pytest.mark.only_asyncio
diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py
index 361e91382..489b70e74 100644
--- a/tests/test_downloader_handler_twisted_ftp.py
+++ b/tests/test_downloader_handler_twisted_ftp.py
@@ -59,7 +59,7 @@ class TestFTPBase(ABC):
port = reactor.listenTCP(0, factory, interface="127.0.0.1")
portno = port.getHost().port
- yield f"https://127.0.0.1:{portno}/"
+ yield f"ftp://127.0.0.1:{portno}/"
await port.stopListening()
@@ -142,15 +142,11 @@ class TestFTPBase(ABC):
server_url: str,
dh: FTPDownloadHandler,
) -> None:
- f, local_fname = mkstemp()
- local_fname_path = Path(local_fname)
- os.close(f)
meta = {}
meta.update(self.req_meta)
request = Request(url=server_url + filename, meta=meta)
r = await dh.download_request(request)
assert type(r) is response_class # pylint: disable=unidiomatic-typecheck
- local_fname_path.unlink()
class TestFTP(TestFTPBase):
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 5f79a5453..bea97642e 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -24,6 +24,7 @@ from tests.test_downloader_handlers_http_base import (
TestHttpWithCrawlerBase,
TestMitmProxyBase,
TestRealWebsiteBase,
+ TestSimpleHttpsBase,
)
from tests.utils.decorators import coroutine_test
@@ -156,6 +157,10 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
await download_handler.download_request(request)
+class TestSimpleHttp2(H2DownloadHandlerMixin, TestSimpleHttpsBase):
+ pass
+
+
class TestHttp2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase):
pass
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index eadb7740e..e685f607a 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -191,17 +191,14 @@ class TestS3:
@contextlib.contextmanager
def _mocked_date(self, date):
- try:
- import botocore.auth # noqa: F401,PLC0415
- except ImportError:
+ import botocore.auth # noqa: F401,PLC0415
+
+ # We need to mock botocore.auth.formatdate, because otherwise
+ # botocore overrides Date header with current date and time
+ # and Authorization header is different each time
+ with mock.patch("botocore.auth.formatdate") as mock_formatdate:
+ mock_formatdate.return_value = date
yield
- else:
- # We need to mock botocore.auth.formatdate, because otherwise
- # botocore overrides Date header with current date and time
- # and Authorization header is different each time
- with mock.patch("botocore.auth.formatdate") as mock_formatdate:
- mock_formatdate.return_value = date
- yield
@coroutine_test
async def test_request_signing1(self):
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index 0e1ff07c9..223f288bf 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -563,7 +563,7 @@ class TestHttpBase(ABC):
) -> None:
request = Request(mockserver.url("/text", is_secure=self.is_secure))
- # 10 is minimal size for this request and the limit is only counted on
+ # 5 is minimal size for this request and the limit is only counted on
# response body. (regardless of headers)
async with self.get_dh({"DOWNLOAD_MAXSIZE": 5}) as download_handler:
response = await download_handler.download_request(request)
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 548c0d8ee..e5d726764 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -203,7 +203,7 @@ class DummyPolicyTestMixin(PolicyTestMixin):
assert mw.process_request(req) is None
# s3 scheme response is cached by default
- req, res = Request("s3://bucket/key"), Response("http://bucket/key")
+ req, res = Request("s3://bucket/key"), Response("s3://bucket/key")
with self._middleware() as mw:
assert mw.process_request(req) is None
mw.process_response(req, res)
@@ -214,7 +214,7 @@ class DummyPolicyTestMixin(PolicyTestMixin):
assert "cached" in cached.flags
# ignore s3 scheme
- req, res = Request("s3://bucket/key2"), Response("http://bucket/key2")
+ req, res = Request("s3://bucket/key2"), Response("s3://bucket/key2")
with self._middleware(HTTPCACHE_IGNORE_SCHEMES=["s3"]) as mw:
assert mw.process_request(req) is None
mw.process_response(req, res)
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index bb7fcd6c7..30caa094f 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -361,7 +361,7 @@ class TestHttpCompression:
zf.write(plainbody)
zf.close()
response = Response(
- "http;//www.example.com/", headers=headers, body=f.getvalue()
+ "http://www.example.com/", headers=headers, body=f.getvalue()
)
request = Request("http://www.example.com/")
@@ -386,7 +386,7 @@ class TestHttpCompression:
zf.write(plainbody)
zf.close()
response = HtmlResponse(
- "http;//www.example.com/page.html", headers=headers, body=f.getvalue()
+ "http://www.example.com/page.html", headers=headers, body=f.getvalue()
)
request = Request("http://www.example.com/")
@@ -493,7 +493,7 @@ class TestHttpCompression:
gz_resp.close()
response = Response(
- "http;//www.example.com/", headers=headers, body=r.getvalue()
+ "http://www.example.com/", headers=headers, body=r.getvalue()
)
request = Request("http://www.example.com/")
diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.py
index 44ade93b7..32935769f 100644
--- a/tests/test_downloadermiddleware_redirect_base.py
+++ b/tests/test_downloadermiddleware_redirect_base.py
@@ -122,7 +122,7 @@ class Base:
req1 = Request("http://a.example/first")
rsp1 = self.get_response(req1, "/redirected")
req2 = self.mw.process_response(req1, rsp1)
- rsp2 = self.get_response(req1, "/redirected2")
+ rsp2 = self.get_response(req2, "/redirected2")
req3 = self.mw.process_response(req2, rsp2)
assert req2.url == "http://a.example/redirected"
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 67af4264c..cf7b614c4 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -16,7 +16,7 @@ class TestDownloaderStats:
self.crawler.stats.open_spider()
self.req = Request("http://scrapytest.org")
- self.res = Response("scrapytest.org", status=400)
+ self.res = Response("http://scrapytest.org", status=400)
def assertStatsEqual(self, key, value):
assert self.crawler.stats.get_value(key) == value, str(
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index a717b18a6..9d58a6e09 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -5,7 +5,6 @@ import pytest
from scrapy import Request
from scrapy.core.downloader import Downloader, Slot
-from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
@@ -65,9 +64,6 @@ class TestCrawl:
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
- def setup_method(self):
- self.runner = CrawlerRunner()
-
@inline_callbacks_test
def test_delay(self):
crawler = get_crawler(DownloaderSlotsSettingsTestSpider)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index b38bf9570..412a59fcd 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -86,7 +86,7 @@ class TestRFPDupeFilter:
df.close("finished")
df2 = _get_dupefilter(settings={"JOBDIR": path}, open_=False)
- assert df != df2
+ assert df is not df2
try:
df2.open()
assert df2.request_seen(r1)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 2cd583721..e51eb4664 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -6,7 +6,6 @@ import subprocess
import sys
from collections import defaultdict
from dataclasses import dataclass
-from logging import DEBUG
from typing import TYPE_CHECKING, Any, cast
from unittest.mock import Mock, call
from urllib.parse import urlparse
@@ -395,6 +394,7 @@ class TestEngine(TestEngineBase):
self._assert_downloaded_responses(run, count=9)
self._assert_scraped_items(run)
self._assert_signals_caught(run)
+ self._assert_headers_received(run)
self._assert_bytes_received(run)
@coroutine_test
@@ -606,7 +606,7 @@ class TestEngineDownload(TestEngineDownloadAsync):
@coroutine_test
-async def test_request_scheduled_signal(caplog):
+async def test_request_scheduled_signal():
class TestScheduler(BaseScheduler):
def __init__(self):
self.enqueued = []
@@ -633,7 +633,6 @@ async def test_request_scheduled_signal(caplog):
keep_request = Request("https://keep.example")
engine._schedule_request(keep_request)
drop_request = Request("https://drop.example")
- caplog.set_level(DEBUG)
engine._schedule_request(drop_request)
assert scheduler.enqueued == [keep_request], (
f"{scheduler.enqueued!r} != [{keep_request!r}]"
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 27e0c6445..c1d6f04eb 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -32,7 +32,6 @@ from scrapy.extensions.feedexport import (
FeedSlot,
FileFeedStorage,
IFeedStorage,
- S3FeedStorage,
)
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
@@ -499,8 +498,7 @@ class TestFeedExport(TestFeedExportBase):
},
}
crawler = get_crawler(ItemSpider, settings)
- with mock.patch.object(S3FeedStorage, "store"):
- yield crawler.crawl(mockserver=self.mockserver)
+ yield crawler.crawl(mockserver=self.mockserver)
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index d855d0f74..0a926479b 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -315,7 +315,7 @@ class TestBatchDeliveries(TestFeedExportBase):
}
data = await self.exported_data(items, settings)
for fmt, expected in formats.items():
- for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
+ for expected_batch, got_batch in zip(expected, data[fmt], strict=True):
assert got_batch == expected_batch
@coroutine_test
@@ -339,7 +339,7 @@ class TestBatchDeliveries(TestFeedExportBase):
}
data = await self.exported_data(items, settings)
for fmt, expected in formats.items():
- for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
+ for expected_batch, got_batch in zip(expected, data[fmt], strict=True):
assert got_batch == expected_batch
@coroutine_test
@@ -447,7 +447,7 @@ class TestBatchDeliveries(TestFeedExportBase):
yield crawler.crawl()
assert len(CustomS3FeedStorage.stubs) == len(items)
- for stub in CustomS3FeedStorage.stubs[:-1]:
+ for stub in CustomS3FeedStorage.stubs:
stub.assert_no_pending_responses()
assert (
"feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats()
diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py
index fa1c0586a..6ebcab152 100644
--- a/tests/test_feedexport_postprocess.py
+++ b/tests/test_feedexport_postprocess.py
@@ -270,7 +270,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
self._named_tempfile("check_CHECK_NONE"): lzma.compress(
self.expected, check=lzma.CHECK_NONE
),
- self._named_tempfile("check_CHECK_CRC256"): lzma.compress(
+ self._named_tempfile("CHECK_SHA256"): lzma.compress(
self.expected, check=lzma.CHECK_SHA256
),
}
@@ -282,7 +282,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_check": lzma.CHECK_NONE,
},
- self._named_tempfile("check_CHECK_CRC256"): {
+ self._named_tempfile("CHECK_SHA256"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_check": lzma.CHECK_SHA256,
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index cec5d728b..28f306e31 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -141,7 +141,7 @@ class Dataloss(LeafResource):
class NoContentLengthHeader(LeafResource):
def render_GET(self, request: TxRequest):
- request.requestHeaders.removeHeader("Content-Length")
+ request.responseHeaders.removeHeader("Content-Length")
self.deferRequest(request, 0, self._delayed_render, request)
return NOT_DONE_YET
@@ -460,9 +460,7 @@ class TestHttps2ClientProtocol:
def test_invalid_negotiated_protocol(
self, server_port: int, client: H2ClientProtocol
) -> Generator[Deferred[Any], Any, None]:
- with mock.patch(
- "scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"
- ):
+ with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", new=b"not-h2"):
request = Request(url=self.get_url(server_port, "/status?n=200"))
with pytest.raises(ResponseFailed):
yield make_request_dfd(client, request)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index fed5dbab7..fd494504d 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -23,7 +23,6 @@ class TestRequest:
# url argument must be basestring
with pytest.raises(TypeError):
self.request_class(123)
- r = self.request_class("http://www.example.com")
r = self.request_class("http://www.example.com")
assert isinstance(r.url, str)
@@ -211,11 +210,11 @@ class TestRequest:
r1.cb_kwargs["key"] = "value"
r2 = r1.copy()
- # make sure copy does not propagate callbacks
+ # make sure callbaclks are copied
assert r1.callback is somecallback
assert r1.errback is somecallback
assert r2.callback is r1.callback
- assert r2.errback is r2.errback
+ assert r2.errback is r1.errback
# make sure flags list is shallow copied
assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 09c95dc29..079c547c7 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,13 +1,19 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
import pytest
-from packaging.version import Version as parse_version
-from w3lib import __version__ as w3lib_version
from w3lib.encoding import resolve_encoding
from scrapy.exceptions import NotSupported
from scrapy.http import Headers, Request, Response
from scrapy.link import Link
+from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS
from tests import get_testdata
+if TYPE_CHECKING:
+ from collections.abc import Iterable
+
class TestResponse:
response_class = Response
@@ -249,7 +255,7 @@ class TestResponse:
r.follow(None)
@pytest.mark.xfail(
- parse_version(w3lib_version) < parse_version("2.1.1"),
+ not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
)
@@ -257,7 +263,7 @@ class TestResponse:
self._assert_followed_url("foo ", "http://example.com/foo")
@pytest.mark.xfail(
- parse_version(w3lib_version) < parse_version("2.1.1"),
+ not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
)
@@ -325,16 +331,26 @@ class TestResponse:
with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
+ @pytest.mark.xfail(
+ not W3LIB_STRIPS_URLS,
+ reason="https://github.com/scrapy/w3lib/pull/207",
+ strict=True,
+ )
def test_follow_all_whitespace(self):
relative = ["foo ", "bar ", "foo/bar ", "bar/foo "]
absolute = [
- "http://example.com/foo%20",
- "http://example.com/bar%20",
- "http://example.com/foo/bar%20",
- "http://example.com/bar/foo%20",
+ "http://example.com/foo",
+ "http://example.com/bar",
+ "http://example.com/foo/bar",
+ "http://example.com/bar/foo",
]
self._assert_followed_all_urls(relative, absolute)
+ @pytest.mark.xfail(
+ not W3LIB_STRIPS_URLS,
+ reason="https://github.com/scrapy/w3lib/pull/207",
+ strict=True,
+ )
def test_follow_all_whitespace_links(self):
absolute = [
"http://example.com/foo ",
@@ -342,8 +358,8 @@ class TestResponse:
"http://example.com/foo/bar ",
"http://example.com/bar/foo ",
]
- links = map(Link, absolute)
- expected = [u.replace(" ", "%20") for u in absolute]
+ links = [Link(u) for u in absolute]
+ expected = [u.strip() for u in absolute]
self._assert_followed_all_urls(links, expected)
def test_follow_all_flags(self):
@@ -357,25 +373,36 @@ class TestResponse:
for req in fol:
assert req.flags == ["cached", "allowed"]
- def _assert_followed_url(self, follow_obj, target_url, response=None):
+ def _assert_followed_url(
+ self,
+ follow_obj: str | Link,
+ target_url: str,
+ response: Response | None = None,
+ encoding: str | None = None,
+ ) -> None:
if response is None:
response = self._links_response()
req = response.follow(follow_obj)
assert req.url == target_url
- return req
+ if encoding is not None:
+ assert req.encoding == encoding
- def _assert_followed_all_urls(self, follow_obj, target_urls, response=None):
+ def _assert_followed_all_urls(
+ self,
+ follow_obj: Iterable[str | Link],
+ target_urls: Iterable[str],
+ response: Response | None = None,
+ ) -> None:
if response is None:
response = self._links_response()
followed = response.follow_all(follow_obj)
- for req, target in zip(followed, target_urls, strict=False):
+ for req, target in zip(followed, target_urls, strict=True):
assert req.url == target
- yield req
- def _links_response(self):
+ def _links_response(self) -> Response:
body = get_testdata("link_extractor", "linkextractor.html")
return self.response_class("http://example.com/index", body=body)
- def _links_response_no_href(self):
+ def _links_response_no_href(self) -> Response:
body = get_testdata("link_extractor", "linkextractor_no_href.html")
return self.response_class("http://example.com/index", body=body)
diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index c16af52b9..4b3fa2302 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -179,7 +179,6 @@ class TestTextResponse(TestResponse):
# Inferring encoding from body also cache decoded body as sideeffect,
# this test tries to ensure that calling response.encoding and
# response.text in indistinct order doesn't affect final
- # response.text in indistinct order doesn't affect final
# values for encoding and decoded body.
url = "http://example.com"
body = b"\xef\xbb\xbfWORD"
@@ -308,11 +307,12 @@ class TestTextResponse(TestResponse):
"http://example.com/sample3.html#foo",
"http://www.google.com/something",
"http://example.com/innertag.html",
+ "http://example.com/page%204.html",
]
# select elements
for sellist in [resp.css("a"), resp.xpath("//a")]:
- for sel, url in zip(sellist, urls, strict=False):
+ for sel, url in zip(sellist, urls, strict=True):
self._assert_followed_url(sel, url, response=resp)
# select elements
@@ -324,7 +324,7 @@ class TestTextResponse(TestResponse):
# href attributes should work
for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]:
- for sel, url in zip(sellist, urls, strict=False):
+ for sel, url in zip(sellist, urls, strict=True):
self._assert_followed_url(sel, url, response=resp)
# non-a elements are not supported
@@ -376,12 +376,12 @@ class TestTextResponse(TestResponse):
encoding="utf8",
body='click me'.encode(),
)
- req = self._assert_followed_url(
+ self._assert_followed_url(
resp1.css("a")[0],
"http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82",
response=resp1,
+ encoding="utf8",
)
- assert req.encoding == "utf8"
resp2 = self.response_class(
"http://example.com",
@@ -390,12 +390,12 @@ class TestTextResponse(TestResponse):
"cp1251"
),
)
- req = self._assert_followed_url(
+ self._assert_followed_url(
resp2.css("a")[0],
"http://example.com/foo?%EF%F0%E8%E2%E5%F2",
response=resp2,
+ encoding="cp1251",
)
- assert req.encoding == "cp1251"
def test_follow_flags(self):
res = self.response_class("http://example.com/")
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 9806315b4..360aa613e 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -28,14 +28,14 @@ class TestLogFormatter:
self.spider = Spider("default")
self.spider.crawler = get_crawler()
- def test_crawled_with_referer(self):
+ def test_crawled_without_referer(self):
req = Request("http://www.example.com")
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws["msg"] % logkws["args"]
assert logline == "Crawled (200) (referer: None)"
- def test_crawled_without_referer(self):
+ def test_crawled_with_referer(self):
req = Request(
"http://www.example.com", headers={"referer": "http://example.com"}
)
@@ -198,7 +198,7 @@ class TestLogformatterSubclass(TestLogFormatter):
self.spider = Spider("default")
self.spider.crawler = get_crawler(Spider)
- def test_crawled_with_referer(self):
+ def test_crawled_without_referer(self):
req = Request("http://www.example.com")
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
@@ -207,7 +207,7 @@ class TestLogformatterSubclass(TestLogFormatter):
logline == "Crawled (200) (referer: None) []"
)
- def test_crawled_without_referer(self):
+ def test_crawled_with_referer(self):
req = Request(
"http://www.example.com",
headers={"referer": "http://example.com"},
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 44df0bdd4..da1bfa317 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -175,10 +175,6 @@ class MockedMediaPipeline(UserDefinedPipeline):
super().__init__(*args, crawler=crawler, **kwargs)
self._mockcalled = []
- def download(self, request, info):
- self._mockcalled.append("download")
- return super().download(request, info)
-
def media_to_download(self, request, info, *, item=None):
self._mockcalled.append("media_to_download")
if "result" in request.meta:
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 29b23496a..5249736f2 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,5 +1,3 @@
-import sys
-
import pytest
from scrapy.robotstxt import (
@@ -8,6 +6,7 @@ from scrapy.robotstxt import (
RerpRobotParser,
decode_robotstxt,
)
+from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
def rerp_available() -> bool:
@@ -139,28 +138,25 @@ class TestDecodeRobotsTxt:
class TestPythonRobotParser(BaseRobotParserTest):
- # https://github.com/python/cpython/pull/149374 improves it
- IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5)
-
def setup_method(self):
super()._setUp(PythonRobotParser)
@pytest.mark.skipif(
- not IMPROVED_ROBOTFILEPARSER,
+ not STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support length based directives precedence.",
)
def test_length_based_precedence(self):
super().test_length_based_precedence()
@pytest.mark.skipif(
- IMPROVED_ROBOTFILEPARSER,
+ STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support order based directives precedence.",
)
def test_order_based_precedence(self):
super().test_order_based_precedence()
@pytest.mark.skipif(
- not IMPROVED_ROBOTFILEPARSER,
+ not STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support wildcards.",
)
def test_allowed_wildcards(self):
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
index db023e1f8..08acacae7 100644
--- a/tests/test_scheduler_base.py
+++ b/tests/test_scheduler_base.py
@@ -104,7 +104,7 @@ class TestMinimalScheduler(InterfaceCheckMixin):
for url in URLS:
assert self.scheduler.enqueue_request(Request(url))
assert not self.scheduler.enqueue_request(Request(url))
- assert self.scheduler.has_pending_requests
+ assert self.scheduler.has_pending_requests()
dequeued = []
while self.scheduler.has_pending_requests():
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index a0d296553..e5891474b 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -238,16 +238,6 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
yield item
-class ProcessSpiderOutputNonIterableMiddleware:
- def process_spider_output(self, response, result):
- return
-
-
-class ProcessSpiderOutputCoroutineMiddleware:
- async def process_spider_output(self, response, result):
- return result
-
-
class ProcessStartSimpleMiddleware:
async def process_start(self, start):
async for item_or_request in start:
@@ -423,20 +413,12 @@ class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
ITEM_TYPE = dict
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
- MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware
MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIteratorMiddleware
def _callback(self) -> Any:
1 / 0
- async def _test_asyncgen_nodowngrade(self, *mw_classes: type[Any]) -> None:
- with pytest.raises(
- _InvalidOutput,
- match=r"Async iterable returned from .+ cannot be downgraded",
- ):
- await self._get_middleware_result(*mw_classes)
-
@coroutine_test
async def test_exc_simple(self):
"""Simple exc mw"""
diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py
index c907c6d73..21df73a65 100644
--- a/tests/test_spidermiddleware_process_start.py
+++ b/tests/test_spidermiddleware_process_start.py
@@ -14,7 +14,6 @@ from .utils.decorators import coroutine_test
ITEM_A = {"id": "a"}
ITEM_B = {"id": "b"}
ITEM_C = {"id": "c"}
-ITEM_D = {"id": "d"}
class AsyncioSleepSpiderMiddleware:
@@ -47,10 +46,6 @@ class ModernWrapSpider(Spider):
yield ITEM_B
-class ModernWrapSpiderSubclass(ModernWrapSpider):
- name = "test"
-
-
class ModernWrapSpiderMiddleware:
async def process_start(self, start):
yield ITEM_A
@@ -79,10 +74,6 @@ class TestMain:
expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C]
await self._test([spider_middleware], spider_cls, expected_items)
- async def _test_douple_wrap(self, smw1, smw2, spider_cls, expected_items=None):
- expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C]
- await self._test([smw1, smw2], spider_cls, expected_items)
-
@coroutine_test
async def test_modern_mw_modern_spider(self):
with warnings.catch_warnings():
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 7431ea6ac..a9089419a 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -836,7 +836,7 @@ class TestRequestMetaSettingFallback:
request_meta,
policy_class,
check_warning,
- ) in self.params[3:]:
+ ) in self.params:
mw = RefererMiddleware(Settings(settings))
response = Response(origin, headers=response_headers)
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index 491fc88f7..e44cfca90 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -1,4 +1,7 @@
+from __future__ import annotations
+
from datetime import datetime, timezone
+from typing import TYPE_CHECKING
import pytest
@@ -7,8 +10,11 @@ from scrapy.extensions.spiderstate import SpiderState
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
+if TYPE_CHECKING:
+ from pathlib import Path
-def test_store_load(tmp_path):
+
+def test_store_load(tmp_path: Path) -> None:
jobdir = str(tmp_path)
spider = Spider(name="default")
@@ -16,6 +22,7 @@ def test_store_load(tmp_path):
ss = SpiderState(jobdir)
ss.spider_opened(spider)
+ assert hasattr(spider, "state")
spider.state["one"] = 1
spider.state["dt"] = dt
ss.spider_closed(spider)
@@ -23,21 +30,23 @@ def test_store_load(tmp_path):
spider2 = Spider(name="default")
ss2 = SpiderState(jobdir)
ss2.spider_opened(spider2)
- assert spider.state == {"one": 1, "dt": dt}
+ assert hasattr(spider2, "state")
+ assert spider2.state == {"one": 1, "dt": dt}
ss2.spider_closed(spider2)
-def test_state_attribute():
+def test_state_attribute() -> None:
# state attribute must be present if jobdir is not set, to provide a
# consistent interface
spider = Spider(name="default")
ss = SpiderState()
ss.spider_opened(spider)
+ assert hasattr(spider, "state")
assert spider.state == {}
ss.spider_closed(spider)
-def test_not_configured():
+def test_not_configured() -> None:
crawler = get_crawler(Spider)
with pytest.raises(NotConfigured):
SpiderState.from_crawler(crawler)
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index a871a282e..5532b4a31 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -83,6 +83,7 @@ class TestParallelAsyncio:
max_parallel_count,
)
assert list(range(length)) == sorted(results)
+ assert parallel_count[0] == 0
assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
@coroutine_test
@@ -101,6 +102,7 @@ class TestParallelAsyncio:
max_parallel_count,
)
assert list(range(length)) == sorted(results)
+ assert parallel_count[0] == 0
assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index c5425d99d..5ea6f678e 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -22,9 +22,7 @@ class NewName(SomeBaseClass):
class TestWarnWhenSubclassed:
- def _mywarnings(
- self, w: list[WarningMessage], category: type[Warning] = MyWarning
- ) -> list[WarningMessage]:
+ def _mywarnings(self, w: list[WarningMessage]) -> list[WarningMessage]:
return [x for x in w if x.category is MyWarning]
def test_no_warning_on_definition(self):
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index 9bd86f7fb..0eb330461 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -17,4 +17,4 @@ def test_urlparse_cached():
assert req1a == urlp
assert req1a is req1b
assert req1a is not req2
- assert req1a is not req2
+ assert req1b is not req2
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 8e5020022..ee552df64 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -76,15 +76,16 @@ class TestLogCounterHandler:
@pytest.fixture
def logger(self, crawler: Crawler) -> Generator[logging.Logger]:
logger = logging.getLogger("test")
- logger.setLevel(logging.NOTSET)
+ logger.setLevel(logging.DEBUG)
logger.propagate = False
- handler = LogCounterHandler(crawler)
+ handler = LogCounterHandler(crawler, level=crawler.settings.get("LOG_LEVEL"))
logger.addHandler(handler)
-
- yield logger
-
- logger.propagate = True
- logger.removeHandler(handler)
+ try:
+ yield logger
+ finally:
+ logger.propagate = True
+ logger.setLevel(logging.NOTSET)
+ logger.removeHandler(handler)
def test_init(self, crawler: Crawler, logger: logging.Logger) -> None:
assert crawler.stats
@@ -102,7 +103,7 @@ class TestLogCounterHandler:
def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None:
logger.debug("test log msg")
assert crawler.stats
- assert crawler.stats.get_value("log_count/INFO") is None
+ assert crawler.stats.get_value("log_count/DEBUG") is None
class TestStreamLogger:
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index e02bdfb69..4544cd29e 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -23,7 +23,7 @@ def _read_browser_output(burl: str):
def test_open_in_browser():
- url = "http:///www.example.com/some/page.html"
+ url = "http://www.example.com/some/page.html"
body = (
b" test page test body "
)