@@ -248,13 +240,14 @@ class TestJMESPath:
r"(\d+)"
) == ["18", "32", "22", "25"]
- @pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath")
- def test_jmespath_not_available(self) -> None:
- body = """
- {
- "website": {"name": "Example"}
- }
- """
- resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")
- with pytest.raises(AttributeError):
- resp.jmespath("website.name").get()
+
+@pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath")
+def test_jmespath_not_available() -> None:
+ body = """
+ {
+ "website": {"name": "Example"}
+ }
+ """
+ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")
+ with pytest.raises(AttributeError):
+ resp.jmespath("website.name").get()
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index d7d900546..e97e114d6 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,10 +1,12 @@
# pylint: disable=unsubscriptable-object,unsupported-membership-test,use-implicit-booleaness-not-comparison
# (too many false positives)
+import warnings
from unittest import mock
import pytest
+from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.settings import (
SETTINGS_PRIORITIES,
BaseSettings,
@@ -12,6 +14,8 @@ from scrapy.settings import (
SettingsAttribute,
get_settings_priority,
)
+from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.test import get_crawler
from . import default_settings
@@ -307,7 +311,7 @@ class TestBaseSettings:
assert settings.getdict("TEST_DICT3", {"key1": 5}) == {"key1": 5}
with pytest.raises(
ValueError,
- match="dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected",
+ match=r"dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected",
):
settings.getdict("TEST_LIST1")
with pytest.raises(
@@ -446,12 +450,8 @@ class TestSettings:
assert mydict["key"] == "val"
def test_passing_objects_as_values(self):
- from scrapy.core.downloader.handlers.file import FileDownloadHandler
- from scrapy.utils.misc import build_from_crawler
- from scrapy.utils.test import get_crawler
-
class TestPipeline:
- def process_item(self, i, s):
+ def process_item(self, i):
return i
settings = Settings(
@@ -471,7 +471,7 @@ class TestSettings:
assert priority == 800
assert mypipeline == TestPipeline
assert isinstance(mypipeline(), TestPipeline)
- assert mypipeline().process_item("item", None) == "item"
+ assert mypipeline().process_item("item") == "item"
myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp")
assert myhandler == FileDownloadHandler
@@ -559,6 +559,17 @@ def test_remove_from_list(before, name, item, after):
assert settings.getpriority(name) == expected_settings.getpriority(name)
+def test_deprecated_concurrent_requests_per_ip_setting():
+ with warnings.catch_warnings(record=True) as warns:
+ settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1})
+ settings.get("CONCURRENT_REQUESTS_PER_IP")
+
+ assert (
+ str(warns[0].message)
+ == "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead."
+ )
+
+
class Component1:
pass
diff --git a/tests/test_signals.py b/tests/test_signals.py
index b20a949e8..2e4f9ffb1 100644
--- a/tests/test_signals.py
+++ b/tests/test_signals.py
@@ -1,11 +1,10 @@
import pytest
from twisted.internet.defer import inlineCallbacks
-from twisted.trial.unittest import TestCase
from scrapy import Request, Spider, signals
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
-from tests.mockserver import MockServer
+from tests.mockserver.http import MockServer
class ItemSpider(Spider):
@@ -21,7 +20,7 @@ class ItemSpider(Spider):
return {"index": response.meta["index"]}
-class TestMain(TestCase):
+class TestMain:
@deferred_f_from_coro_f
async def test_scheduler_empty(self):
crawler = get_crawler()
@@ -35,17 +34,17 @@ class TestMain(TestCase):
assert len(calls) >= 1
-class TestMockServer(TestCase):
+class TestMockServer:
@classmethod
- def setUpClass(cls):
+ def setup_class(cls):
cls.mockserver = MockServer()
cls.mockserver.__enter__()
@classmethod
- def tearDownClass(cls):
+ def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
- def setUp(self):
+ def setup_method(self):
self.items = []
async def _on_item_scraped(self, item):
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 4e4a99638..dcc9d1e82 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -1,5 +1,9 @@
+from __future__ import annotations
+
import gzip
+import re
import warnings
+from datetime import datetime
from io import BytesIO
from logging import ERROR, WARNING
from pathlib import Path
@@ -9,7 +13,6 @@ from unittest import mock
import pytest
from testfixtures import LogCapture
from twisted.internet.defer import inlineCallbacks
-from twisted.trial import unittest
from w3lib.url import safe_url_string
from scrapy import signals
@@ -31,15 +34,9 @@ from scrapy.utils.test import get_crawler, get_reactor_settings
from tests import get_testdata, tests_datadir
-class TestSpider(unittest.TestCase):
+class TestSpider:
spider_class = Spider
- def setUp(self):
- warnings.simplefilter("always")
-
- def tearDown(self):
- warnings.resetwarnings()
-
def test_base_spider(self):
spider = self.spider_class("example.com")
assert spider.name == "example.com"
@@ -234,7 +231,7 @@ class TestCSVFeedSpider(TestSpider):
class TestCrawlSpider(TestSpider):
- test_body = b"""
@@ -295,8 +292,6 @@ class TestCrawlSpider(TestSpider):
)
class _CrawlSpider(self.spider_class):
- import re
-
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="filter_process_links"),)
@@ -531,7 +526,7 @@ class TestSitemapSpider(TestSpider):
g.close()
GZBODY = f.getvalue()
- def assertSitemapBody(self, response, body):
+ def assertSitemapBody(self, response: Response, body: bytes | None) -> None:
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
assert spider._get_sitemap_body(response) == body
@@ -633,8 +628,6 @@ Sitemap: /sitemap-relative-url.xml
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
- from datetime import datetime
-
for entry in entries:
date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
if date_time.year > 2008:
@@ -704,8 +697,6 @@ Sitemap: /sitemap-relative-url.xml
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
- from datetime import datetime
-
for entry in entries:
date_time = datetime.strptime(
entry["lastmod"].split("T")[0], "%Y-%m-%d"
@@ -735,6 +726,7 @@ Sitemap: /sitemap-relative-url.xml
response = Response(url="https://example.com", body=body, request=request)
assert spider._get_sitemap_body(response) is None
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr(self):
class DownloadMaxSizeSpider(self.spider_class):
download_maxsize = 10_000_000
@@ -782,6 +774,7 @@ Sitemap: /sitemap-relative-url.xml
),
)
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_download_warnsize_spider_attr(self):
class DownloadWarnSizeSpider(self.spider_class):
download_warnsize = 10_000_000
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index 3c7fc65d5..d4eca85b8 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -1,9 +1,11 @@
+from __future__ import annotations
+
import warnings
from asyncio import sleep
+from typing import Any
import pytest
from testfixtures import LogCapture
-from twisted.trial.unittest import TestCase
from scrapy import Spider, signals
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -18,8 +20,10 @@ ITEM_A = {"id": "a"}
ITEM_B = {"id": "b"}
-class TestMain(TestCase):
- async def _test_spider(self, spider, expected_items=None):
+class TestMain:
+ async def _test_spider(
+ self, spider: type[Spider], expected_items: list[Any] | None = None
+ ) -> None:
actual_items = []
expected_items = [] if expected_items is None else expected_items
@@ -29,6 +33,7 @@ class TestMain(TestCase):
crawler = get_crawler(spider)
crawler.signals.connect(track_item, signals.item_scraped)
await maybe_deferred_to_future(crawler.crawl())
+ assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 28ffbe767..3ac6d9af7 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -2,43 +2,50 @@ from __future__ import annotations
from collections.abc import AsyncIterator, Iterable
from inspect import isasyncgen
-from typing import Any
+from typing import TYPE_CHECKING, Any
from unittest import mock
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
-from twisted.trial.unittest import TestCase
from scrapy.core.spidermw import SpiderMiddlewareManager
-from scrapy.exceptions import _InvalidOutput
+from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.asyncgen import collect_asyncgen
-from scrapy.utils.defer import (
- deferred_f_from_coro_f,
- maybe_deferred_to_future,
-)
+from scrapy.utils.asyncio import call_later
+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
+if TYPE_CHECKING:
+ from twisted.python.failure import Failure
-class TestSpiderMiddleware(TestCase):
- def setUp(self):
+ from scrapy.crawler import Crawler
+
+
+class TestSpiderMiddleware:
+ def setup_method(self):
self.request = Request("http://example.com/index.html")
self.response = Response(self.request.url, request=self.request)
self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES_BASE": {}})
- self.spider = self.crawler._create_spider("foo")
+ self.crawler.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
async def _scrape_response(self) -> Any:
- """Execute spider mw manager's scrape_response method and return the result.
+ """Execute spider mw manager's scrape_response_async method and return the result.
Raise exception in case of failure.
"""
- scrape_func = mock.MagicMock()
- return await maybe_deferred_to_future(
- self.mwman.scrape_response(
- scrape_func, self.response, self.request, self.spider
- )
+
+ def scrape_func(
+ response: Response | Failure, request: Request
+ ) -> defer.Deferred[Iterable[Any]]:
+ it = mock.MagicMock()
+ return defer.succeed(it)
+
+ return await self.mwman.scrape_response_async(
+ scrape_func, self.response, self.request
)
@@ -48,7 +55,7 @@ class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_invalid_process_spider_input(self):
class InvalidProcessSpiderInputMiddleware:
- def process_spider_input(self, response, spider):
+ def process_spider_input(self, response):
return 1
self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())
@@ -62,7 +69,7 @@ class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_invalid_process_spider_output(self):
class InvalidProcessSpiderOutputMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
return 1
self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())
@@ -76,11 +83,11 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_invalid_process_spider_exception(self):
class InvalidProcessSpiderOutputExceptionMiddleware:
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
return 1
class RaiseExceptionProcessSpiderOutputMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
raise RuntimeError
self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())
@@ -95,11 +102,11 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_process_spider_exception_return_none(self):
class ProcessSpiderExceptionReturnNoneMiddleware:
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
return None
class RaiseExceptionProcessSpiderOutputMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
1 / 0
self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
@@ -118,30 +125,42 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects
@staticmethod
- def _construct_mw_setting(*mw_classes, start_index: int | None = None):
+ def _construct_mw_setting(
+ *mw_classes: type[Any], start_index: int | None = None
+ ) -> dict[type[Any], int]:
if start_index is None:
start_index = 10
return {i: c for c, i in enumerate(mw_classes, start=start_index)}
- def _scrape_func(self, *args, **kwargs):
+ def _callback(self) -> Any:
yield {"foo": 1}
yield {"foo": 2}
yield {"foo": 3}
- async def _get_middleware_result(self, *mw_classes, start_index: int | None = None):
+ async def _scrape_func(
+ self, response: Response | Failure, request: Request
+ ) -> Iterable[Any] | AsyncIterator[Any]:
+ return self._callback()
+
+ async def _get_middleware_result(
+ self, *mw_classes: type[Any], start_index: int | None = None
+ ) -> Any:
setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
self.crawler = get_crawler(
Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting}
)
- self.spider = self.crawler._create_spider("foo")
+ self.crawler.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
return await self.mwman.scrape_response_async(
- self._scrape_func, self.response, self.request, self.spider
+ self._scrape_func, self.response, self.request
)
async def _test_simple_base(
- self, *mw_classes, downgrade: bool = False, start_index: int | None = None
- ):
+ self,
+ *mw_classes: type[Any],
+ downgrade: bool = False,
+ start_index: int | None = None,
+ ) -> None:
with LogCapture() as log:
result = await self._get_middleware_result(
*mw_classes, start_index=start_index
@@ -156,8 +175,11 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
)
async def _test_asyncgen_base(
- self, *mw_classes, downgrade: bool = False, start_index: int | None = None
- ):
+ self,
+ *mw_classes: type[Any],
+ downgrade: bool = False,
+ start_index: int | None = None,
+ ) -> None:
with LogCapture() as log:
result = await self._get_middleware_result(
*mw_classes, start_index=start_index
@@ -170,39 +192,37 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
class ProcessSpiderOutputSimpleMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
yield from result
class ProcessSpiderOutputAsyncGenMiddleware:
- async def process_spider_output(self, response, result, spider):
+ async def process_spider_output(self, response, result):
async for r in result:
yield r
class ProcessSpiderOutputUniversalMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
yield from result
- async def process_spider_output_async(self, response, result, spider):
+ async def process_spider_output_async(self, response, result):
async for r in result:
yield r
class ProcessSpiderExceptionSimpleIterableMiddleware:
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
yield {"foo": 1}
yield {"foo": 2}
yield {"foo": 3}
class ProcessSpiderExceptionAsyncIteratorMiddleware:
- async def process_spider_exception(self, response, exception, spider):
+ async def process_spider_exception(self, response, exception):
yield {"foo": 1}
d = defer.Deferred()
- from twisted.internet import reactor
-
- reactor.callLater(0, d.callback, None)
+ call_later(0, d.callback, None)
await maybe_deferred_to_future(d)
yield {"foo": 2}
yield {"foo": 3}
@@ -265,8 +285,8 @@ class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware):
class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
"""process_spider_output tests for async generator callbacks"""
- async def _scrape_func(self, *args, **kwargs):
- for item in super()._scrape_func():
+ async def _callback(self) -> Any:
+ for item in super()._callback():
yield item
@deferred_f_from_coro_f
@@ -296,12 +316,12 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
class ProcessSpiderOutputNonIterableMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
return
class ProcessSpiderOutputCoroutineMiddleware:
- async def process_spider_output(self, response, result, spider):
+ async def process_spider_output(self, response, result):
return result
@@ -335,7 +355,9 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):
ITEM_TYPE = (Request, dict)
MW_SIMPLE = ProcessStartSimpleMiddleware
- async def _get_processed_start(self, *mw_classes):
+ async def _get_processed_start(
+ self, *mw_classes: type[Any]
+ ) -> AsyncIterator[Any] | None:
class TestSpider(Spider):
name = "test"
@@ -348,9 +370,9 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):
self.crawler = get_crawler(
TestSpider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting}
)
- self.spider = self.crawler._create_spider()
+ self.crawler.spider = self.crawler._create_spider()
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
- return await self.mwman.process_start(self.spider)
+ return await self.mwman.process_start()
@deferred_f_from_coro_f
async def test_simple(self):
@@ -363,82 +385,90 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):
class UniversalMiddlewareNoSync:
- async def process_spider_output_async(self, response, result, spider):
+ async def process_spider_output_async(self, response, result):
yield
class UniversalMiddlewareBothSync:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
yield
- def process_spider_output_async(self, response, result, spider):
+ def process_spider_output_async(self, response, result):
yield
class UniversalMiddlewareBothAsync:
- async def process_spider_output(self, response, result, spider):
+ async def process_spider_output(self, response, result):
yield
- async def process_spider_output_async(self, response, result, spider):
+ async def process_spider_output_async(self, response, result):
yield
class TestUniversalMiddlewareManager:
- def setup_method(self):
- self.mwman = SpiderMiddlewareManager()
+ @pytest.fixture
+ def crawler(self) -> Crawler:
+ return get_crawler(Spider)
- def test_simple_mw(self):
+ @pytest.fixture
+ def mwman(self, crawler: Crawler) -> SpiderMiddlewareManager:
+ return SpiderMiddlewareManager.from_crawler(crawler)
+
+ def test_simple_mw(self, mwman: SpiderMiddlewareManager) -> None:
mw = ProcessSpiderOutputSimpleMiddleware()
- self.mwman._add_middleware(mw)
+ mwman._add_middleware(mw)
assert (
- self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
+ mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
)
- def test_async_mw(self):
+ def test_async_mw(self, mwman: SpiderMiddlewareManager) -> None:
mw = ProcessSpiderOutputAsyncGenMiddleware()
- self.mwman._add_middleware(mw)
+ mwman._add_middleware(mw)
assert (
- self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
+ mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
)
- def test_universal_mw(self):
+ def test_universal_mw(self, mwman: SpiderMiddlewareManager) -> None:
mw = ProcessSpiderOutputUniversalMiddleware()
- self.mwman._add_middleware(mw)
- assert self.mwman.methods["process_spider_output"][0] == (
+ mwman._add_middleware(mw)
+ assert mwman.methods["process_spider_output"][0] == (
mw.process_spider_output,
mw.process_spider_output_async,
)
- def test_universal_mw_no_sync(self):
- with LogCapture() as log:
- self.mwman._add_middleware(UniversalMiddlewareNoSync())
+ def test_universal_mw_no_sync(
+ self, mwman: SpiderMiddlewareManager, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ mwman._add_middleware(UniversalMiddlewareNoSync())
assert (
"UniversalMiddlewareNoSync has process_spider_output_async"
- " without process_spider_output" in str(log)
+ " without process_spider_output" in caplog.text
)
- assert self.mwman.methods["process_spider_output"][0] is None
+ assert mwman.methods["process_spider_output"][0] is None
- def test_universal_mw_both_sync(self):
+ def test_universal_mw_both_sync(
+ self, mwman: SpiderMiddlewareManager, caplog: pytest.LogCaptureFixture
+ ) -> None:
mw = UniversalMiddlewareBothSync()
- with LogCapture() as log:
- self.mwman._add_middleware(mw)
+ mwman._add_middleware(mw)
assert (
"UniversalMiddlewareBothSync.process_spider_output_async "
- "is not an async generator function" in str(log)
+ "is not an async generator function" in caplog.text
)
assert (
- self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
+ mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable
)
- def test_universal_mw_both_async(self):
- with LogCapture() as log:
- self.mwman._add_middleware(UniversalMiddlewareBothAsync())
+ def test_universal_mw_both_async(
+ self, mwman: SpiderMiddlewareManager, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ mwman._add_middleware(UniversalMiddlewareBothAsync())
assert (
"UniversalMiddlewareBothAsync.process_spider_output "
"is an async generator function while process_spider_output_async exists"
- in str(log)
+ in caplog.text
)
- assert self.mwman.methods["process_spider_output"][0] is None
+ assert mwman.methods["process_spider_output"][0] is None
class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
@@ -447,13 +477,15 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
- async def _get_middleware_result(self, *mw_classes, start_index: int | None = None):
+ async def _get_middleware_result(
+ self, *mw_classes: type[Any], start_index: int | None = None
+ ) -> Any:
setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES": setting})
- self.spider = self.crawler._create_spider("foo")
+ self.crawler.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
return await self.mwman.scrape_response_async(
- self._scrape_func, self.response, self.request, self.spider
+ self._scrape_func, self.response, self.request
)
@deferred_f_from_coro_f
@@ -488,8 +520,8 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
- async def _scrape_func(self, *args, **kwargs):
- for item in super()._scrape_func():
+ async def _callback(self) -> Any:
+ for item in super()._callback():
yield item
@deferred_f_from_coro_f
@@ -531,12 +563,13 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware
MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIteratorMiddleware
- def _scrape_func(self, *args, **kwargs):
+ def _callback(self) -> Any:
1 / 0
- async def _test_asyncgen_nodowngrade(self, *mw_classes):
+ async def _test_asyncgen_nodowngrade(self, *mw_classes: type[Any]) -> None:
with pytest.raises(
- _InvalidOutput, match="Async iterable returned from .+ cannot be downgraded"
+ _InvalidOutput,
+ match=r"Async iterable returned from .+ cannot be downgraded",
):
await self._get_middleware_result(*mw_classes)
@@ -569,3 +602,57 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
async def test_exc_async_simple(self):
"""Async exc mw -> simple output mw; cannot work as downgrading is not supported"""
await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN)
+
+
+class TestDeprecatedSpiderArg(TestSpiderMiddleware):
+ @deferred_f_from_coro_f
+ async def test_deprecated_mw_spider_arg(self):
+ class DeprecatedSpiderArgMiddleware:
+ def process_spider_input(self, response, spider):
+ return None
+
+ def process_spider_output(self, response, result, spider):
+ 1 / 0
+
+ def process_spider_exception(self, response, exception, spider):
+ return []
+
+ with (
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"process_spider_input\(\) requires a spider argument",
+ ),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"process_spider_output\(\) requires a spider argument",
+ ),
+ pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"process_spider_exception\(\) requires a spider argument",
+ ),
+ ):
+ self.mwman._add_middleware(DeprecatedSpiderArgMiddleware())
+ await self._scrape_response()
+
+ @deferred_f_from_coro_f
+ async def test_deprecated_mwman_spider_arg(self):
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a spider argument to SpiderMiddlewareManager.process_start\(\)"
+ r" is deprecated and the passed value is ignored",
+ ):
+ await self.mwman.process_start(DefaultSpider())
+
+ @deferred_f_from_coro_f
+ async def test_deprecated_mwman_spider_arg_no_crawler(self):
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"MiddlewareManager.__init__\(\) was called without the crawler argument",
+ ):
+ mwman = SpiderMiddlewareManager()
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a spider argument to SpiderMiddlewareManager.process_start\(\)"
+ r" is deprecated, SpiderMiddlewareManager should be instantiated with a Crawler",
+ ):
+ await mwman.process_start(DefaultSpider())
diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py
index 77d055d50..da570904d 100644
--- a/tests/test_spidermiddleware_base.py
+++ b/tests/test_spidermiddleware_base.py
@@ -18,7 +18,7 @@ def crawler() -> Crawler:
return get_crawler(Spider)
-def test_trivial(crawler):
+def test_trivial(crawler: Crawler) -> None:
class TrivialSpiderMiddleware(BaseSpiderMiddleware):
pass
@@ -28,15 +28,13 @@ def test_trivial(crawler):
test_req = Request("data:,")
spider_output = [test_req, {"foo": "bar"}]
for processed in [
- list(
- mw.process_spider_output(Response("data:,"), spider_output, crawler.spider)
- ),
- list(mw.process_start_requests(spider_output, crawler.spider)),
+ list(mw.process_spider_output(Response("data:,"), spider_output)),
+ list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
]:
assert processed == [test_req, {"foo": "bar"}]
-def test_processed_request(crawler):
+def test_processed_request(crawler: Crawler) -> None:
class ProcessReqSpiderMiddleware(BaseSpiderMiddleware):
def get_processed_request(
self, request: Request, response: Response | None
@@ -53,10 +51,8 @@ def test_processed_request(crawler):
test_req3 = Request("data:3,")
spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]
for processed in [
- list(
- mw.process_spider_output(Response("data:,"), spider_output, crawler.spider)
- ),
- list(mw.process_start_requests(spider_output, crawler.spider)),
+ list(mw.process_spider_output(Response("data:,"), spider_output)),
+ list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
]:
assert len(processed) == 3
assert isinstance(processed[0], Request)
@@ -66,7 +62,7 @@ def test_processed_request(crawler):
assert processed[2].url == "data:30,"
-def test_processed_item(crawler):
+def test_processed_item(crawler: Crawler) -> None:
class ProcessItemSpiderMiddleware(BaseSpiderMiddleware):
def get_processed_item(self, item: Any, response: Response | None) -> Any:
if item["foo"] == 2:
@@ -79,15 +75,13 @@ def test_processed_item(crawler):
test_req = Request("data:,")
spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]
for processed in [
- list(
- mw.process_spider_output(Response("data:,"), spider_output, crawler.spider)
- ),
- list(mw.process_start_requests(spider_output, crawler.spider)),
+ list(mw.process_spider_output(Response("data:,"), spider_output)),
+ list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
]:
assert processed == [{"foo": 1}, test_req, {"foo": 30}]
-def test_processed_both(crawler):
+def test_processed_both(crawler: Crawler) -> None:
class ProcessBothSpiderMiddleware(BaseSpiderMiddleware):
def get_processed_request(
self, request: Request, response: Response | None
@@ -118,10 +112,8 @@ def test_processed_both(crawler):
test_req3,
]
for processed in [
- list(
- mw.process_spider_output(Response("data:,"), spider_output, crawler.spider)
- ),
- list(mw.process_start_requests(spider_output, crawler.spider)),
+ list(mw.process_spider_output(Response("data:,"), spider_output)),
+ list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
]:
assert len(processed) == 4
assert isinstance(processed[0], Request)
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index 9b4aa624c..00e547305 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -1,38 +1,57 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.depth import DepthMiddleware
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
+if TYPE_CHECKING:
+ from collections.abc import Generator
-class TestDepthMiddleware:
- def setup_method(self):
- crawler = get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True})
- self.spider = crawler._create_spider("scrapytest.org")
+ from scrapy.crawler import Crawler
+ from scrapy.statscollectors import StatsCollector
- self.stats = crawler.stats
- self.stats.open_spider(self.spider)
- self.mw = DepthMiddleware.from_crawler(crawler)
+@pytest.fixture
+def crawler() -> Crawler:
+ return get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True})
- def test_process_spider_output(self):
- req = Request("http://scrapytest.org")
- resp = Response("http://scrapytest.org")
- resp.request = req
- result = [Request("http://scrapytest.org")]
- out = list(self.mw.process_spider_output(resp, result, self.spider))
- assert out == result
+@pytest.fixture
+def stats(crawler: Crawler) -> Generator[StatsCollector]:
+ assert crawler.stats is not None
+ crawler.stats.open_spider()
- rdc = self.stats.get_value("request_depth_count/1", spider=self.spider)
- assert rdc == 1
+ yield crawler.stats
- req.meta["depth"] = 1
+ crawler.stats.close_spider()
- out2 = list(self.mw.process_spider_output(resp, result, self.spider))
- assert not out2
- rdm = self.stats.get_value("request_depth_max", spider=self.spider)
- assert rdm == 1
+@pytest.fixture
+def mw(crawler: Crawler) -> DepthMiddleware:
+ return DepthMiddleware.from_crawler(crawler)
- def teardown_method(self):
- self.stats.close_spider(self.spider, "")
+
+def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> None:
+ req = Request("http://scrapytest.org")
+ resp = Response("http://scrapytest.org")
+ resp.request = req
+ result = [Request("http://scrapytest.org")]
+
+ out = list(mw.process_spider_output(resp, result))
+ assert out == result
+
+ rdc = stats.get_value("request_depth_count/1")
+ assert rdc == 1
+
+ req.meta["depth"] = 1
+
+ out2 = list(mw.process_spider_output(resp, result))
+ assert not out2
+
+ rdm = stats.get_value("request_depth_max")
+ assert rdm == 1
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 12dbaaa96..9899a0cd7 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -1,16 +1,16 @@
+from __future__ import annotations
+
import logging
import pytest
from testfixtures import LogCapture
from twisted.internet.defer import inlineCallbacks
-from twisted.trial.unittest import TestCase
from scrapy.http import Request, Response
-from scrapy.settings import Settings
from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware
-from scrapy.spiders import Spider
+from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
-from tests.mockserver import MockServer
+from tests.mockserver.http import MockServer
from tests.spiders import MockServerSpider
@@ -49,111 +49,132 @@ class _HttpErrorSpider(MockServerSpider):
return failure
-def _responses(request, status_codes):
- responses = []
- for code in status_codes:
- response = Response(request.url, status=code)
- response.request = request
- responses.append(response)
- return responses
+req = Request("http://scrapytest.org")
+
+
+def _response(request: Request, status_code: int) -> Response:
+ return Response(request.url, status=status_code, request=request)
+
+
+@pytest.fixture
+def res200() -> Response:
+ return _response(req, 200)
+
+
+@pytest.fixture
+def res402() -> Response:
+ return _response(req, 402)
+
+
+@pytest.fixture
+def res404() -> Response:
+ return _response(req, 404)
class TestHttpErrorMiddleware:
- def setup_method(self):
- crawler = get_crawler(Spider)
- self.spider = Spider.from_crawler(crawler, name="foo")
- self.mw = HttpErrorMiddleware(Settings({}))
- self.req = Request("http://scrapytest.org")
- self.res200, self.res404 = _responses(self.req, [200, 404])
+ @pytest.fixture
+ def mw(self) -> HttpErrorMiddleware:
+ crawler = get_crawler(DefaultSpider)
+ crawler.spider = crawler._create_spider()
+ return HttpErrorMiddleware.from_crawler(crawler)
- def test_process_spider_input(self):
- assert self.mw.process_spider_input(self.res200, self.spider) is None
+ def test_process_spider_input(
+ self, mw: HttpErrorMiddleware, res200: Response, res404: Response
+ ) -> None:
+ mw.process_spider_input(res200)
with pytest.raises(HttpError):
- self.mw.process_spider_input(self.res404, self.spider)
+ mw.process_spider_input(res404)
- def test_process_spider_exception(self):
- assert (
- self.mw.process_spider_exception(
- self.res404, HttpError(self.res404), self.spider
- )
- == []
- )
- assert (
- self.mw.process_spider_exception(self.res404, Exception(), self.spider)
- is None
- )
+ def test_process_spider_exception(
+ self, mw: HttpErrorMiddleware, res404: Response
+ ) -> None:
+ assert mw.process_spider_exception(res404, HttpError(res404)) == []
+ assert mw.process_spider_exception(res404, Exception()) is None
- def test_handle_httpstatus_list(self):
- res = self.res404.copy()
- res.request = Request(
+ def test_handle_httpstatus_list(
+ self, mw: HttpErrorMiddleware, res404: Response
+ ) -> None:
+ request = Request(
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
)
- assert self.mw.process_spider_input(res, self.spider) is None
+ res = _response(request, 404)
+ mw.process_spider_input(res)
- self.spider.handle_httpstatus_list = [404]
- assert self.mw.process_spider_input(self.res404, self.spider) is None
+ assert mw.crawler.spider
+ mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
+ mw.process_spider_input(res404)
class TestHttpErrorMiddlewareSettings:
"""Similar test, but with settings"""
- def setup_method(self):
- self.spider = Spider("foo")
- self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)}))
- self.req = Request("http://scrapytest.org")
- self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
+ @pytest.fixture
+ def mw(self) -> HttpErrorMiddleware:
+ crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOWED_CODES": (402,)})
+ crawler.spider = crawler._create_spider()
+ return HttpErrorMiddleware.from_crawler(crawler)
- def test_process_spider_input(self):
- assert self.mw.process_spider_input(self.res200, self.spider) is None
+ def test_process_spider_input(
+ self,
+ mw: HttpErrorMiddleware,
+ res200: Response,
+ res402: Response,
+ res404: Response,
+ ) -> None:
+ mw.process_spider_input(res200)
with pytest.raises(HttpError):
- self.mw.process_spider_input(self.res404, self.spider)
- assert self.mw.process_spider_input(self.res402, self.spider) is None
+ mw.process_spider_input(res404)
+ mw.process_spider_input(res402)
- def test_meta_overrides_settings(self):
+ def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None:
request = Request(
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
)
- res404 = self.res404.copy()
- res404.request = request
- res402 = self.res402.copy()
- res402.request = request
+ res404 = _response(request, 404)
+ res402 = _response(request, 402)
- assert self.mw.process_spider_input(res404, self.spider) is None
+ mw.process_spider_input(res404)
with pytest.raises(HttpError):
- self.mw.process_spider_input(res402, self.spider)
+ mw.process_spider_input(res402)
- def test_spider_override_settings(self):
- self.spider.handle_httpstatus_list = [404]
- assert self.mw.process_spider_input(self.res404, self.spider) is None
+ def test_spider_override_settings(
+ self, mw: HttpErrorMiddleware, res402: Response, res404: Response
+ ) -> None:
+ assert mw.crawler.spider
+ mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
+ mw.process_spider_input(res404)
with pytest.raises(HttpError):
- self.mw.process_spider_input(self.res402, self.spider)
+ mw.process_spider_input(res402)
class TestHttpErrorMiddlewareHandleAll:
- def setup_method(self):
- self.spider = Spider("foo")
- self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))
- self.req = Request("http://scrapytest.org")
- self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
+ @pytest.fixture
+ def mw(self) -> HttpErrorMiddleware:
+ crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOW_ALL": True})
+ crawler.spider = crawler._create_spider()
+ return HttpErrorMiddleware.from_crawler(crawler)
- def test_process_spider_input(self):
- assert self.mw.process_spider_input(self.res200, self.spider) is None
- assert self.mw.process_spider_input(self.res404, self.spider) is None
+ def test_process_spider_input(
+ self,
+ mw: HttpErrorMiddleware,
+ res200: Response,
+ res404: Response,
+ ) -> None:
+ mw.process_spider_input(res200)
+ mw.process_spider_input(res404)
- def test_meta_overrides_settings(self):
+ def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None:
request = Request(
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
)
- res404 = self.res404.copy()
- res404.request = request
- res402 = self.res402.copy()
- res402.request = request
+ res404 = _response(request, 404)
+ res402 = _response(request, 402)
- assert self.mw.process_spider_input(res404, self.spider) is None
+ mw.process_spider_input(res404)
with pytest.raises(HttpError):
- self.mw.process_spider_input(res402, self.spider)
+ mw.process_spider_input(res402)
- def test_httperror_allow_all_false(self):
+ def test_httperror_allow_all_false(self) -> None:
crawler = get_crawler(_HttpErrorSpider)
mw = HttpErrorMiddleware.from_crawler(crawler)
request_httpstatus_false = Request(
@@ -162,31 +183,29 @@ class TestHttpErrorMiddlewareHandleAll:
request_httpstatus_true = Request(
"http://scrapytest.org", meta={"handle_httpstatus_all": True}
)
- res404 = self.res404.copy()
- res404.request = request_httpstatus_false
- res402 = self.res402.copy()
- res402.request = request_httpstatus_true
+ res404 = _response(request_httpstatus_false, 404)
+ res402 = _response(request_httpstatus_true, 402)
with pytest.raises(HttpError):
- mw.process_spider_input(res404, self.spider)
- assert mw.process_spider_input(res402, self.spider) is None
+ mw.process_spider_input(res404)
+ mw.process_spider_input(res402)
-class TestHttpErrorMiddlewareIntegrational(TestCase):
+class TestHttpErrorMiddlewareIntegrational:
@classmethod
- def setUpClass(cls):
+ def setup_class(cls):
cls.mockserver = MockServer()
cls.mockserver.__enter__()
@classmethod
- def tearDownClass(cls):
+ def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
@inlineCallbacks
def test_middleware_works(self):
crawler = get_crawler(_HttpErrorSpider)
yield crawler.crawl(mockserver=self.mockserver)
- assert not crawler.spider.skipped, crawler.spider.skipped
+ assert not crawler.spider.skipped
assert crawler.spider.parsed == {"200"}
assert crawler.spider.failed == {"404", "402", "500"}
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
deleted file mode 100644
index e4f4b8f9b..000000000
--- a/tests/test_spidermiddleware_offsite.py
+++ /dev/null
@@ -1,105 +0,0 @@
-import warnings
-from urllib.parse import urlparse
-
-from scrapy.http import Request, Response
-from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, PortWarning, URLWarning
-from scrapy.spiders import Spider
-from scrapy.utils.test import get_crawler
-
-
-class TestOffsiteMiddleware:
- def setup_method(self):
- crawler = get_crawler(Spider)
- self.spider = crawler.spider = crawler._create_spider(**self._get_spiderargs())
- self.mw = OffsiteMiddleware.from_crawler(crawler)
- self.mw.spider_opened(self.spider)
-
- def _get_spiderargs(self):
- return {
- "name": "foo",
- "allowed_domains": ["scrapytest.org", "scrapy.org", "scrapy.test.org"],
- }
-
- def test_process_spider_output(self):
- res = Response("http://scrapytest.org")
-
- onsite_reqs = [
- Request("http://scrapytest.org/1"),
- Request("http://scrapy.org/1"),
- Request("http://sub.scrapy.org/1"),
- Request("http://offsite.tld/letmepass", dont_filter=True),
- Request("http://offsite-2.tld/allow", meta={"allow_offsite": True}),
- Request("http://scrapy.test.org/"),
- Request("http://scrapy.test.org:8000/"),
- ]
- offsite_reqs = [
- Request("http://scrapy2.org"),
- Request("http://offsite.tld/"),
- Request("http://offsite.tld/scrapytest.org"),
- Request("http://offsite.tld/rogue.scrapytest.org"),
- Request("http://rogue.scrapytest.org.haha.com"),
- Request("http://roguescrapytest.org"),
- Request("http://test.org/"),
- Request("http://notscrapy.test.org/"),
- ]
- reqs = onsite_reqs + offsite_reqs
-
- out = list(self.mw.process_spider_output(res, reqs, self.spider))
- assert out == onsite_reqs
-
-
-class TestOffsiteMiddleware2(TestOffsiteMiddleware):
- def _get_spiderargs(self):
- return {"name": "foo", "allowed_domains": None}
-
- def test_process_spider_output(self):
- res = Response("http://scrapytest.org")
- reqs = [Request("http://a.com/b.html"), Request("http://b.com/1")]
- out = list(self.mw.process_spider_output(res, reqs, self.spider))
- assert out == reqs
-
-
-class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
- def _get_spiderargs(self):
- return {"name": "foo"}
-
-
-class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
- def _get_spiderargs(self):
- bad_hostname = urlparse("http:////scrapytest.org").hostname
- return {
- "name": "foo",
- "allowed_domains": ["scrapytest.org", None, bad_hostname],
- }
-
- def test_process_spider_output(self):
- res = Response("http://scrapytest.org")
- reqs = [Request("http://scrapytest.org/1")]
- out = list(self.mw.process_spider_output(res, reqs, self.spider))
- assert out == reqs
-
-
-class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
- def test_get_host_regex(self):
- self.spider.allowed_domains = [
- "http://scrapytest.org",
- "scrapy.org",
- "scrapy.test.org",
- ]
- with warnings.catch_warnings(record=True) as w:
- warnings.simplefilter("always")
- self.mw.get_host_regex(self.spider)
- assert issubclass(w[-1].category, URLWarning)
-
-
-class TestOffsiteMiddleware6(TestOffsiteMiddleware4):
- def test_get_host_regex(self):
- self.spider.allowed_domains = [
- "scrapytest.org:8000",
- "scrapy.org",
- "scrapy.test.org",
- ]
- with warnings.catch_warnings(record=True) as w:
- warnings.simplefilter("always")
- self.mw.get_host_regex(self.spider)
- assert issubclass(w[-1].category, PortWarning)
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 60464d696..1808e087c 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -1,24 +1,32 @@
from testfixtures import LogCapture
-from twisted.trial.unittest import TestCase
from scrapy import Request, Spider
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.test import get_crawler
-from tests.mockserver import MockServer
+from tests.mockserver.http import MockServer
-class LogExceptionMiddleware:
- def process_spider_exception(self, response, exception, spider):
- spider.logger.info(
+class _BaseSpiderMiddleware:
+ def __init__(self, crawler):
+ self.crawler = crawler
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
+
+class LogExceptionMiddleware(_BaseSpiderMiddleware):
+ def process_spider_exception(self, response, exception):
+ self.crawler.spider.logger.info(
"Middleware: %s exception caught", exception.__class__.__name__
)
# ================================================================================
# (0) recover from an exception on a spider callback
-class RecoveryMiddleware:
- def process_spider_exception(self, response, exception, spider):
- spider.logger.info(
+class RecoveryMiddleware(_BaseSpiderMiddleware):
+ def process_spider_exception(self, response, exception):
+ self.crawler.spider.logger.info(
"Middleware: %s exception caught", exception.__class__.__name__
)
return [
@@ -56,9 +64,9 @@ class RecoveryAsyncGenSpider(RecoverySpider):
# ================================================================================
# (1) exceptions from a spider middleware's process_spider_input method
-class FailProcessSpiderInputMiddleware:
- def process_spider_input(self, response, spider):
- spider.logger.info("Middleware: will raise IndexError")
+class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware):
+ def process_spider_input(self, response):
+ self.crawler.spider.logger.info("Middleware: will raise IndexError")
raise IndexError
@@ -160,27 +168,31 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS
# ================================================================================
# (4) exceptions from a middleware process_spider_output method (generator)
-class _GeneratorDoNothingMiddleware:
- def process_spider_output(self, response, result, spider):
+class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware):
+ def process_spider_output(self, response, result):
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ self.crawler.spider.logger.info(
+ "%s: %s caught", method, exception.__class__.__name__
+ )
-class GeneratorFailMiddleware:
- def process_spider_output(self, response, result, spider):
+class GeneratorFailMiddleware(_BaseSpiderMiddleware):
+ def process_spider_output(self, response, result):
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r
raise LookupError
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ self.crawler.spider.logger.info(
+ "%s: %s caught", method, exception.__class__.__name__
+ )
yield {"processed": [method]}
@@ -188,15 +200,17 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware):
pass
-class GeneratorRecoverMiddleware:
- def process_spider_output(self, response, result, spider):
+class GeneratorRecoverMiddleware(_BaseSpiderMiddleware):
+ def process_spider_output(self, response, result):
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ self.crawler.spider.logger.info(
+ "%s: %s caught", method, exception.__class__.__name__
+ )
yield {"processed": [method]}
@@ -227,30 +241,34 @@ class GeneratorOutputChainSpider(Spider):
# (5) exceptions from a middleware process_spider_output method (not generator)
-class _NotGeneratorDoNothingMiddleware:
- def process_spider_output(self, response, result, spider):
+class _NotGeneratorDoNothingMiddleware(_BaseSpiderMiddleware):
+ def process_spider_output(self, response, result):
out = []
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r)
return out
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ self.crawler.spider.logger.info(
+ "%s: %s caught", method, exception.__class__.__name__
+ )
-class NotGeneratorFailMiddleware:
- def process_spider_output(self, response, result, spider):
+class NotGeneratorFailMiddleware(_BaseSpiderMiddleware):
+ def process_spider_output(self, response, result):
out = []
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r)
raise ReferenceError
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ self.crawler.spider.logger.info(
+ "%s: %s caught", method, exception.__class__.__name__
+ )
return [{"processed": [method]}]
@@ -258,17 +276,19 @@ class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddlewa
pass
-class NotGeneratorRecoverMiddleware:
- def process_spider_output(self, response, result, spider):
+class NotGeneratorRecoverMiddleware(_BaseSpiderMiddleware):
+ def process_spider_output(self, response, result):
out = []
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r)
return out
- def process_spider_exception(self, response, exception, spider):
+ def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
- spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ self.crawler.spider.logger.info(
+ "%s: %s caught", method, exception.__class__.__name__
+ )
return [{"processed": [method]}]
@@ -298,16 +318,16 @@ class NotGeneratorOutputChainSpider(Spider):
# ================================================================================
-class TestSpiderMiddleware(TestCase):
+class TestSpiderMiddleware:
mockserver: MockServer
@classmethod
- def setUpClass(cls):
+ def setup_class(cls):
cls.mockserver = MockServer()
cls.mockserver.__enter__()
@classmethod
- def tearDownClass(cls):
+ def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
async def crawl_log(self, spider: type[Spider]) -> LogCapture:
diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py
index e1c8b5fec..a525f991d 100644
--- a/tests/test_spidermiddleware_process_start.py
+++ b/tests/test_spidermiddleware_process_start.py
@@ -2,7 +2,6 @@ import warnings
from asyncio import sleep
import pytest
-from twisted.trial.unittest import TestCase
from scrapy import Spider, signals
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -106,7 +105,7 @@ class DeprecatedWrapSpiderMiddleware:
yield ITEM_C
-class TestMain(TestCase):
+class TestMain:
async def _test(self, spider_middlewares, spider_cls, expected_items):
actual_items = []
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 300a40c13..68fcad98c 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import warnings
-from typing import Any
+from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urlparse
import pytest
@@ -31,7 +31,13 @@ from scrapy.spidermiddlewares.referer import (
StrictOriginWhenCrossOriginPolicy,
UnsafeUrlPolicy,
)
-from scrapy.spiders import Spider
+from scrapy.utils.spider import DefaultSpider
+from scrapy.utils.test import get_crawler
+
+if TYPE_CHECKING:
+ from collections.abc import Sequence
+
+ from scrapy.crawler import Crawler
class TestRefererMiddleware:
@@ -42,22 +48,22 @@ class TestRefererMiddleware:
("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"),
]
- def setup_method(self):
- self.spider = Spider("foo")
+ @pytest.fixture
+ def mw(self) -> RefererMiddleware:
settings = Settings(self.settings)
- self.mw = RefererMiddleware(settings)
+ return RefererMiddleware(settings)
- def get_request(self, target):
+ def get_request(self, target: str) -> Request:
return Request(target, meta=self.req_meta)
- def get_response(self, origin):
+ def get_response(self, origin: str) -> Response:
return Response(origin, headers=self.resp_headers)
- def test(self):
+ def test(self, mw: RefererMiddleware) -> None:
for origin, target, referrer in self.scenarii:
response = self.get_response(origin)
request = self.get_request(target)
- out = list(self.mw.process_spider_output(response, [request], self.spider))
+ out = list(mw.process_spider_output(response, [request]))
assert out[0].headers.get("Referer") == referrer
@@ -966,7 +972,7 @@ class TestPolicyHeaderPrecedence004(
class TestReferrerOnRedirect(TestRefererMiddleware):
settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}
- scenarii: list[
+ scenarii: Sequence[
tuple[str, str, tuple[tuple[int, str], ...], bytes | None, bytes | None]
] = [ # type: ignore[assignment]
(
@@ -1002,13 +1008,26 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
),
]
- def setup_method(self):
- self.spider = Spider("foo")
- settings = Settings(self.settings)
- self.referrermw = RefererMiddleware(settings)
- self.redirectmw = RedirectMiddleware(settings)
+ @pytest.fixture
+ def crawler(self) -> Crawler:
+ crawler = get_crawler(DefaultSpider, self.settings)
+ crawler.spider = crawler._create_spider()
+ return crawler
- def test(self):
+ @pytest.fixture
+ def referrermw(self, crawler: Crawler) -> RefererMiddleware:
+ return RefererMiddleware.from_crawler(crawler)
+
+ @pytest.fixture
+ def redirectmw(self, crawler: Crawler) -> RedirectMiddleware:
+ return RedirectMiddleware.from_crawler(crawler)
+
+ def test( # type: ignore[override]
+ self,
+ crawler: Crawler,
+ referrermw: RefererMiddleware,
+ redirectmw: RedirectMiddleware,
+ ) -> None:
for (
parent,
target,
@@ -1019,19 +1038,18 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
response = self.get_response(parent)
request = self.get_request(target)
- out = list(
- self.referrermw.process_spider_output(response, [request], self.spider)
- )
+ out = list(referrermw.process_spider_output(response, [request]))
assert out[0].headers.get("Referer") == init_referrer
for status, url in redirections:
response = Response(
request.url, headers={"Location": url}, status=status
)
- request = self.redirectmw.process_response(
- request, response, self.spider
+ request = cast(
+ "Request", redirectmw.process_response(request, response)
)
- self.referrermw.request_scheduled(request, self.spider)
+ assert crawler.spider
+ referrermw.request_scheduled(request, crawler.spider)
assert isinstance(request, Request)
assert request.headers.get("Referer") == final_referrer
diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py
index 295b10ea8..c2efe47c9 100644
--- a/tests/test_spidermiddleware_start.py
+++ b/tests/test_spidermiddleware_start.py
@@ -1,5 +1,3 @@
-from twisted.trial.unittest import TestCase
-
from scrapy.http import Request
from scrapy.spidermiddlewares.start import StartSpiderMiddleware
from scrapy.spiders import Spider
@@ -8,7 +6,7 @@ from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
-class TestMiddleware(TestCase):
+class TestMiddleware:
@deferred_f_from_coro_f
async def test_async(self):
crawler = get_crawler(Spider)
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index 5cc3cdc6c..750ae3b07 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -1,39 +1,56 @@
-from testfixtures import LogCapture
+from __future__ import annotations
+
+from logging import INFO
+from typing import TYPE_CHECKING
+
+import pytest
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler
+ from scrapy.statscollectors import StatsCollector
-class TestUrlLengthMiddleware:
- def setup_method(self):
- self.maxlength = 25
- crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": self.maxlength})
- self.spider = crawler._create_spider("foo")
- self.stats = crawler.stats
- self.mw = UrlLengthMiddleware.from_crawler(crawler)
- self.response = Response("http://scrapytest.org")
- self.short_url_req = Request("http://scrapytest.org/")
- self.long_url_req = Request("http://scrapytest.org/this_is_a_long_url")
- self.reqs = [self.short_url_req, self.long_url_req]
+maxlength = 25
+response = Response("http://scrapytest.org")
+short_url_req = Request("http://scrapytest.org/")
+long_url_req = Request("http://scrapytest.org/this_is_a_long_url")
+reqs: list[Request] = [short_url_req, long_url_req]
- def process_spider_output(self):
- return list(
- self.mw.process_spider_output(self.response, self.reqs, self.spider)
- )
- def test_middleware_works(self):
- assert self.process_spider_output() == [self.short_url_req]
+@pytest.fixture
+def crawler() -> Crawler:
+ return get_crawler(Spider, {"URLLENGTH_LIMIT": maxlength})
- def test_logging(self):
- with LogCapture() as log:
- self.process_spider_output()
- ric = self.stats.get_value(
- "urllength/request_ignored_count", spider=self.spider
- )
- assert ric == 1
+@pytest.fixture
+def stats(crawler: Crawler) -> StatsCollector:
+ assert crawler.stats is not None
+ return crawler.stats
- assert f"Ignoring link (url length > {self.maxlength})" in str(log)
+
+@pytest.fixture
+def mw(crawler: Crawler) -> UrlLengthMiddleware:
+ return UrlLengthMiddleware.from_crawler(crawler)
+
+
+def process_spider_output(mw: UrlLengthMiddleware) -> list[Request]:
+ return list(mw.process_spider_output(response, reqs))
+
+
+def test_middleware_works(mw: UrlLengthMiddleware) -> None:
+ assert process_spider_output(mw) == [short_url_req]
+
+
+def test_logging(
+ stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture
+) -> None:
+ with caplog.at_level(INFO):
+ process_spider_output(mw)
+ ric = stats.get_value("urllength/request_ignored_count")
+ assert ric == 1
+ assert f"Ignoring link (url length > {maxlength})" in caplog.text
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index cd31891a0..491fc88f7 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -1,6 +1,4 @@
-import shutil
from datetime import datetime, timezone
-from tempfile import mkdtemp
import pytest
@@ -10,37 +8,36 @@ from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
-class TestSpiderState:
- def test_store_load(self):
- jobdir = mkdtemp()
- try:
- spider = Spider(name="default")
- dt = datetime.now(tz=timezone.utc)
+def test_store_load(tmp_path):
+ jobdir = str(tmp_path)
- ss = SpiderState(jobdir)
- ss.spider_opened(spider)
- spider.state["one"] = 1
- spider.state["dt"] = dt
- ss.spider_closed(spider)
+ spider = Spider(name="default")
+ dt = datetime.now(tz=timezone.utc)
- spider2 = Spider(name="default")
- ss2 = SpiderState(jobdir)
- ss2.spider_opened(spider2)
- assert spider.state == {"one": 1, "dt": dt}
- ss2.spider_closed(spider2)
- finally:
- shutil.rmtree(jobdir)
+ ss = SpiderState(jobdir)
+ ss.spider_opened(spider)
+ spider.state["one"] = 1
+ spider.state["dt"] = dt
+ ss.spider_closed(spider)
- def test_state_attribute(self):
- # state attribute must be present if jobdir is not set, to provide a
- # consistent interface
- spider = Spider(name="default")
- ss = SpiderState()
- ss.spider_opened(spider)
- assert spider.state == {}
- ss.spider_closed(spider)
+ spider2 = Spider(name="default")
+ ss2 = SpiderState(jobdir)
+ ss2.spider_opened(spider2)
+ assert spider.state == {"one": 1, "dt": dt}
+ ss2.spider_closed(spider2)
- def test_not_configured(self):
- crawler = get_crawler(Spider)
- with pytest.raises(NotConfigured):
- SpiderState.from_crawler(crawler)
+
+def test_state_attribute():
+ # state attribute must be present if jobdir is not set, to provide a
+ # consistent interface
+ spider = Spider(name="default")
+ ss = SpiderState()
+ ss.spider_opened(spider)
+ assert spider.state == {}
+ ss.spider_closed(spider)
+
+
+def test_not_configured():
+ crawler = get_crawler(Spider)
+ with pytest.raises(NotConfigured):
+ SpiderState.from_crawler(crawler)
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 21bbeece2..8283d3d5d 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -33,13 +33,13 @@ def nonserializable_object_test(self):
q = self.queue()
with pytest.raises(
ValueError,
- match="unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as",
+ match=r"unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as",
):
q.push(lambda x: x)
# Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
sel = Selector(text="
some text
")
with pytest.raises(
- ValueError, match="unmarshallable object|can't pickle Selector objects"
+ ValueError, match=r"unmarshallable object|can't pickle Selector objects"
):
q.push(sel)
@@ -117,7 +117,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
q = self.queue()
with pytest.raises(
ValueError,
- match="Can't (get|pickle) local object|Can't pickle .*: it's not found as",
+ match=r"Can't (get|pickle) local object|Can't pickle .*: it's not found as",
) as exc_info:
q.push(lambda x: x)
if hasattr(sys, "pypy_version_info"):
diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py
index 8353ad73c..847a76ab6 100644
--- a/tests/test_squeues_request.py
+++ b/tests/test_squeues_request.py
@@ -2,7 +2,10 @@
Queues that handle requests
"""
-from pathlib import Path
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
import pytest
import queuelib
@@ -19,86 +22,65 @@ from scrapy.squeues import (
)
from scrapy.utils.test import get_crawler
-
-class TestBaseQueue:
- def setup_method(self):
- self.crawler = get_crawler(Spider)
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler
-class RequestQueueTestMixin:
- def queue(self, base_path: Path):
+HAVE_PEEK = hasattr(queuelib.queue.FifoMemoryQueue, "peek")
+
+
+@pytest.fixture
+def crawler() -> Crawler:
+ return get_crawler(Spider)
+
+
+class TestRequestQueueBase(ABC):
+ @property
+ @abstractmethod
+ def is_fifo(self) -> bool:
raise NotImplementedError
- def test_one_element_with_peek(self, tmp_path):
- if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
+ @pytest.mark.parametrize("test_peek", [True, False])
+ def test_one_element(self, q: queuelib.queue.BaseQueue, test_peek: bool):
+ if test_peek and not HAVE_PEEK:
pytest.skip("The queuelib queues do not define peek")
- q = self.queue(tmp_path)
+ if not test_peek and HAVE_PEEK:
+ pytest.skip("The queuelib queues define peek")
assert len(q) == 0
- assert q.peek() is None
+ if test_peek:
+ assert q.peek() is None
assert q.pop() is None
req = Request("http://www.example.com")
q.push(req)
assert len(q) == 1
- assert q.peek().url == req.url
- assert q.pop().url == req.url
+ if test_peek:
+ result = q.peek()
+ assert result is not None
+ assert result.url == req.url
+ else:
+ with pytest.raises(
+ NotImplementedError,
+ match="The underlying queue class does not implement 'peek'",
+ ):
+ q.peek()
+ result = q.pop()
+ assert result is not None
+ assert result.url == req.url
assert len(q) == 0
- assert q.peek() is None
+ if test_peek:
+ assert q.peek() is None
assert q.pop() is None
q.close()
- def test_one_element_without_peek(self, tmp_path):
- if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
- pytest.skip("The queuelib queues define peek")
- q = self.queue(tmp_path)
- assert len(q) == 0
- assert q.pop() is None
- req = Request("http://www.example.com")
- q.push(req)
- assert len(q) == 1
- with pytest.raises(
- NotImplementedError,
- match="The underlying queue class does not implement 'peek'",
- ):
- q.peek()
- assert q.pop().url == req.url
- assert len(q) == 0
- assert q.pop() is None
- q.close()
-
-
-class FifoQueueMixin(RequestQueueTestMixin):
- def test_fifo_with_peek(self, tmp_path):
- if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
+ @pytest.mark.parametrize("test_peek", [True, False])
+ def test_order(self, q: queuelib.queue.BaseQueue, test_peek: bool):
+ if test_peek and not HAVE_PEEK:
pytest.skip("The queuelib queues do not define peek")
- q = self.queue(tmp_path)
- assert len(q) == 0
- assert q.peek() is None
- assert q.pop() is None
- req1 = Request("http://www.example.com/1")
- req2 = Request("http://www.example.com/2")
- req3 = Request("http://www.example.com/3")
- q.push(req1)
- q.push(req2)
- q.push(req3)
- assert len(q) == 3
- assert q.peek().url == req1.url
- assert q.pop().url == req1.url
- assert len(q) == 2
- assert q.peek().url == req2.url
- assert q.pop().url == req2.url
- assert len(q) == 1
- assert q.peek().url == req3.url
- assert q.pop().url == req3.url
- assert len(q) == 0
- assert q.peek() is None
- assert q.pop() is None
- q.close()
-
- def test_fifo_without_peek(self, tmp_path):
- if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
+ if not test_peek and HAVE_PEEK:
pytest.skip("The queuelib queues define peek")
- q = self.queue(tmp_path)
assert len(q) == 0
+ if test_peek:
+ assert q.peek() is None
assert q.pop() is None
req1 = Request("http://www.example.com/1")
req2 = Request("http://www.example.com/2")
@@ -106,111 +88,80 @@ class FifoQueueMixin(RequestQueueTestMixin):
q.push(req1)
q.push(req2)
q.push(req3)
- with pytest.raises(
- NotImplementedError,
- match="The underlying queue class does not implement 'peek'",
- ):
- q.peek()
- assert len(q) == 3
- assert q.pop().url == req1.url
- assert len(q) == 2
- assert q.pop().url == req2.url
- assert len(q) == 1
- assert q.pop().url == req3.url
+ if not test_peek:
+ with pytest.raises(
+ NotImplementedError,
+ match="The underlying queue class does not implement 'peek'",
+ ):
+ q.peek()
+ reqs = [req1, req2, req3] if self.is_fifo else [req3, req2, req1]
+ for i, req in enumerate(reqs):
+ assert len(q) == 3 - i
+ if test_peek:
+ result = q.peek()
+ assert result is not None
+ assert result.url == req.url
+ result = q.pop()
+ assert result is not None
+ assert result.url == req.url
assert len(q) == 0
+ if test_peek:
+ assert q.peek() is None
assert q.pop() is None
q.close()
-class LifoQueueMixin(RequestQueueTestMixin):
- def test_lifo_with_peek(self, tmp_path):
- if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
- pytest.skip("The queuelib queues do not define peek")
- q = self.queue(tmp_path)
- assert len(q) == 0
- assert q.peek() is None
- assert q.pop() is None
- req1 = Request("http://www.example.com/1")
- req2 = Request("http://www.example.com/2")
- req3 = Request("http://www.example.com/3")
- q.push(req1)
- q.push(req2)
- q.push(req3)
- assert len(q) == 3
- assert q.peek().url == req3.url
- assert q.pop().url == req3.url
- assert len(q) == 2
- assert q.peek().url == req2.url
- assert q.pop().url == req2.url
- assert len(q) == 1
- assert q.peek().url == req1.url
- assert q.pop().url == req1.url
- assert len(q) == 0
- assert q.peek() is None
- assert q.pop() is None
- q.close()
+class TestPickleFifoDiskQueueRequest(TestRequestQueueBase):
+ is_fifo = True
- def test_lifo_without_peek(self, tmp_path):
- if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
- pytest.skip("The queuelib queues define peek")
- q = self.queue(tmp_path)
- assert len(q) == 0
- assert q.pop() is None
- req1 = Request("http://www.example.com/1")
- req2 = Request("http://www.example.com/2")
- req3 = Request("http://www.example.com/3")
- q.push(req1)
- q.push(req2)
- q.push(req3)
- with pytest.raises(
- NotImplementedError,
- match="The underlying queue class does not implement 'peek'",
- ):
- q.peek()
- assert len(q) == 3
- assert q.pop().url == req3.url
- assert len(q) == 2
- assert q.pop().url == req2.url
- assert len(q) == 1
- assert q.pop().url == req1.url
- assert len(q) == 0
- assert q.pop() is None
- q.close()
-
-
-class TestPickleFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue):
- def queue(self, base_path):
+ @pytest.fixture
+ def q(self, crawler, tmp_path):
return PickleFifoDiskQueue.from_crawler(
- crawler=self.crawler, key=str(base_path / "pickle" / "fifo")
+ crawler=crawler, key=str(tmp_path / "pickle" / "fifo")
)
-class TestPickleLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue):
- def queue(self, base_path):
+class TestPickleLifoDiskQueueRequest(TestRequestQueueBase):
+ is_fifo = False
+
+ @pytest.fixture
+ def q(self, crawler, tmp_path):
return PickleLifoDiskQueue.from_crawler(
- crawler=self.crawler, key=str(base_path / "pickle" / "lifo")
+ crawler=crawler, key=str(tmp_path / "pickle" / "lifo")
)
-class TestMarshalFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue):
- def queue(self, base_path):
+class TestMarshalFifoDiskQueueRequest(TestRequestQueueBase):
+ is_fifo = True
+
+ @pytest.fixture
+ def q(self, crawler, tmp_path):
return MarshalFifoDiskQueue.from_crawler(
- crawler=self.crawler, key=str(base_path / "marshal" / "fifo")
+ crawler=crawler, key=str(tmp_path / "marshal" / "fifo")
)
-class TestMarshalLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue):
- def queue(self, base_path):
+class TestMarshalLifoDiskQueueRequest(TestRequestQueueBase):
+ is_fifo = False
+
+ @pytest.fixture
+ def q(self, crawler, tmp_path):
return MarshalLifoDiskQueue.from_crawler(
- crawler=self.crawler, key=str(base_path / "marshal" / "lifo")
+ crawler=crawler, key=str(tmp_path / "marshal" / "lifo")
)
-class TestFifoMemoryQueueRequest(FifoQueueMixin, TestBaseQueue):
- def queue(self, base_path):
- return FifoMemoryQueue.from_crawler(crawler=self.crawler)
+class TestFifoMemoryQueueRequest(TestRequestQueueBase):
+ is_fifo = True
+
+ @pytest.fixture
+ def q(self, crawler):
+ return FifoMemoryQueue.from_crawler(crawler=crawler)
-class TestLifoMemoryQueueRequest(LifoQueueMixin, TestBaseQueue):
- def queue(self, base_path):
- return LifoMemoryQueue.from_crawler(crawler=self.crawler)
+class TestLifoMemoryQueueRequest(TestRequestQueueBase):
+ is_fifo = False
+
+ @pytest.fixture
+ def q(self, crawler):
+ return LifoMemoryQueue.from_crawler(crawler=crawler)
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 537614364..6814a652e 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,28 +1,45 @@
+from __future__ import annotations
+
from datetime import datetime
+from typing import TYPE_CHECKING
from unittest import mock
+import pytest
+
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.corestats import CoreStats
from scrapy.spiders import Spider
from scrapy.statscollectors import DummyStatsCollector, StatsCollector
from scrapy.utils.test import get_crawler
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler
+
+
+@pytest.fixture
+def crawler() -> Crawler:
+ return get_crawler(Spider)
+
+
+@pytest.fixture
+def spider(crawler: Crawler) -> Spider:
+ return crawler._create_spider("foo")
+
class TestCoreStatsExtension:
- def setup_method(self):
- self.crawler = get_crawler(Spider)
- self.spider = self.crawler._create_spider("foo")
-
@mock.patch("scrapy.extensions.corestats.datetime")
- def test_core_stats_default_stats_collector(self, mock_datetime):
+ def test_core_stats_default_stats_collector(
+ self, mock_datetime: mock.Mock, crawler: Crawler, spider: Spider
+ ) -> None:
fixed_datetime = datetime(2019, 12, 1, 11, 38)
mock_datetime.now = mock.Mock(return_value=fixed_datetime)
- self.crawler.stats = StatsCollector(self.crawler)
- ext = CoreStats.from_crawler(self.crawler)
- ext.spider_opened(self.spider)
- ext.item_scraped({}, self.spider)
- ext.response_received(self.spider)
- ext.item_dropped({}, self.spider, ZeroDivisionError())
- ext.spider_closed(self.spider, "finished")
+ crawler.stats = StatsCollector(crawler)
+ ext = CoreStats.from_crawler(crawler)
+ ext.spider_opened(spider)
+ ext.item_scraped({}, spider)
+ ext.response_received(spider)
+ ext.item_dropped({}, spider, ZeroDivisionError())
+ ext.spider_closed(spider, "finished")
assert ext.stats._stats == {
"start_time": fixed_datetime,
"finish_time": fixed_datetime,
@@ -34,24 +51,22 @@ class TestCoreStatsExtension:
"elapsed_time_seconds": 0.0,
}
- def test_core_stats_dummy_stats_collector(self):
- self.crawler.stats = DummyStatsCollector(self.crawler)
- ext = CoreStats.from_crawler(self.crawler)
- ext.spider_opened(self.spider)
- ext.item_scraped({}, self.spider)
- ext.response_received(self.spider)
- ext.item_dropped({}, self.spider, ZeroDivisionError())
- ext.spider_closed(self.spider, "finished")
+ def test_core_stats_dummy_stats_collector(
+ self, crawler: Crawler, spider: Spider
+ ) -> None:
+ crawler.stats = DummyStatsCollector(crawler)
+ ext = CoreStats.from_crawler(crawler)
+ ext.spider_opened(spider)
+ ext.item_scraped({}, spider)
+ ext.response_received(spider)
+ ext.item_dropped({}, spider, ZeroDivisionError())
+ ext.spider_closed(spider, "finished")
assert ext.stats._stats == {}
class TestStatsCollector:
- def setup_method(self):
- self.crawler = get_crawler(Spider)
- self.spider = self.crawler._create_spider("foo")
-
- def test_collector(self):
- stats = StatsCollector(self.crawler)
+ def test_collector(self, crawler: Crawler) -> None:
+ stats = StatsCollector(crawler)
assert stats.get_stats() == {}
assert stats.get_value("anything") is None
assert stats.get_value("anything", "default") == "default"
@@ -77,8 +92,8 @@ class TestStatsCollector:
stats.min_value("test4", 7)
assert stats.get_value("test4") == 7
- def test_dummy_collector(self):
- stats = DummyStatsCollector(self.crawler)
+ def test_dummy_collector(self, crawler: Crawler) -> None:
+ stats = DummyStatsCollector(crawler)
assert stats.get_stats() == {}
assert stats.get_value("anything") is None
assert stats.get_value("anything", "default") == "default"
@@ -86,7 +101,20 @@ class TestStatsCollector:
stats.inc_value("v1")
stats.max_value("v2", 100)
stats.min_value("v3", 100)
- stats.open_spider("a")
- stats.set_value("test", "value", spider=self.spider)
+ stats.open_spider()
+ stats.set_value("test", "value")
assert stats.get_stats() == {}
- assert stats.get_stats("a") == {}
+
+ def test_deprecated_spider_arg(self, crawler: Crawler, spider: Spider) -> None:
+ stats = StatsCollector(crawler)
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a 'spider' argument to StatsCollector.set_value\(\) is deprecated",
+ ):
+ stats.set_value("test", "value", spider=spider)
+ assert stats.get_stats() == {"test": "value"}
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a 'spider' argument to StatsCollector.get_stats\(\) is deprecated",
+ ):
+ assert stats.get_stats(spider) == {"test": "value"}
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index a4f31096e..b7497c608 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -1,31 +1,35 @@
import scrapy
-class TestToplevel:
- def test_version(self):
- assert isinstance(scrapy.__version__, str)
+def test_version():
+ assert isinstance(scrapy.__version__, str)
- def test_version_info(self):
- assert isinstance(scrapy.version_info, tuple)
- def test_request_shortcut(self):
- from scrapy.http import FormRequest, Request
+def test_version_info():
+ assert isinstance(scrapy.version_info, tuple)
- assert scrapy.Request is Request
- assert scrapy.FormRequest is FormRequest
- def test_spider_shortcut(self):
- from scrapy.spiders import Spider
+def test_request_shortcut():
+ from scrapy.http import FormRequest, Request # noqa: PLC0415
- assert scrapy.Spider is Spider
+ assert scrapy.Request is Request
+ assert scrapy.FormRequest is FormRequest
- def test_selector_shortcut(self):
- from scrapy.selector import Selector
- assert scrapy.Selector is Selector
+def test_spider_shortcut():
+ from scrapy.spiders import Spider # noqa: PLC0415
- def test_item_shortcut(self):
- from scrapy.item import Field, Item
+ assert scrapy.Spider is Spider
- assert scrapy.Item is Item
- assert scrapy.Field is Field
+
+def test_selector_shortcut():
+ from scrapy.selector import Selector # noqa: PLC0415
+
+ assert scrapy.Selector is Selector
+
+
+def test_item_shortcut():
+ from scrapy.item import Field, Item # noqa: PLC0415
+
+ assert scrapy.Item is Item
+ assert scrapy.Field is Field
diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py
deleted file mode 100644
index 0e1e89e81..000000000
--- a/tests/test_urlparse_monkeypatches.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from urllib.parse import urlparse
-
-
-class TestUrlparse:
- def test_s3_url(self):
- p = urlparse("s3://bucket/key/name?param=value")
- assert p.scheme == "s3"
- assert p.hostname == "bucket"
- assert p.path == "/key/name"
- assert p.query == "param=value"
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index 9b5a25b3a..b2d2b04c7 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -1,10 +1,8 @@
-from twisted.trial import unittest
-
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import deferred_f_from_coro_f
-class TestAsyncgenUtils(unittest.TestCase):
+class TestAsyncgenUtils:
@deferred_f_from_coro_f
async def test_as_async_generator(self):
ag = as_async_generator(range(42))
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index a6e52eb26..b6b77c9e3 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -7,7 +7,6 @@ from unittest import mock
import pytest
from twisted.internet.defer import Deferred
-from twisted.trial import unittest
from scrapy.utils.asyncgen import as_async_generator
from scrapy.utils.asyncio import (
@@ -21,15 +20,14 @@ if TYPE_CHECKING:
from collections.abc import AsyncGenerator
-@pytest.mark.usefixtures("reactor_pytest")
class TestAsyncio:
- def test_is_asyncio_available(self):
+ def test_is_asyncio_available(self, reactor_pytest: str) -> None:
# the result should depend only on the pytest --reactor argument
- assert is_asyncio_available() == (self.reactor_pytest != "default")
+ assert is_asyncio_available() == (reactor_pytest == "asyncio")
@pytest.mark.only_asyncio
-class TestParallelAsyncio(unittest.TestCase):
+class TestParallelAsyncio:
"""Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync."""
CONCURRENT_ITEMS = 50
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 26f158380..b6a9d8b06 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -31,7 +31,7 @@ class TestBuildComponentList:
# Same priority raises ValueError
duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20)
with pytest.raises(
- ValueError, match="Some paths in .* convert to the same object"
+ ValueError, match=r"Some paths in .* convert to the same object"
):
build_component_list(duplicate_bs, convert=lambda x: x.lower())
@@ -47,12 +47,11 @@ class TestBuildComponentList:
assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"]
-class TestUtilsConf:
- def test_arglist_to_dict(self):
- assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == {
- "arg1": "val1",
- "arg2": "val2",
- }
+def test_arglist_to_dict():
+ assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == {
+ "arg1": "val1",
+ "arg2": "val2",
+ }
class TestFeedExportConfig:
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
index 6598bdce7..dc1d96f66 100644
--- a/tests/test_utils_console.py
+++ b/tests/test_utils_console.py
@@ -18,23 +18,24 @@ except ImportError:
ipy = False
-class TestUtilsConsole:
- def test_get_shell_embed_func(self):
- shell = get_shell_embed_func(["invalid"])
- assert shell is None
+def test_get_shell_embed_func():
+ shell = get_shell_embed_func(["invalid"])
+ assert shell is None
- shell = get_shell_embed_func(["invalid", "python"])
- assert callable(shell)
- assert shell.__name__ == "_embed_standard_shell"
+ shell = get_shell_embed_func(["invalid", "python"])
+ assert callable(shell)
+ assert shell.__name__ == "_embed_standard_shell"
- @pytest.mark.skipif(not bpy, reason="bpython not available in testenv")
- def test_get_shell_embed_func2(self):
- shell = get_shell_embed_func(["bpython"])
- assert callable(shell)
- assert shell.__name__ == "_embed_bpython_shell"
- @pytest.mark.skipif(not ipy, reason="IPython not available in testenv")
- def test_get_shell_embed_func3(self):
- # default shell should be 'ipython'
- shell = get_shell_embed_func()
- assert shell.__name__ == "_embed_ipython_shell"
+@pytest.mark.skipif(not bpy, reason="bpython not available in testenv")
+def test_get_shell_embed_func_bpython():
+ shell = get_shell_embed_func(["bpython"])
+ assert callable(shell)
+ assert shell.__name__ == "_embed_bpython_shell"
+
+
+@pytest.mark.skipif(not ipy, reason="IPython not available in testenv")
+def test_get_shell_embed_func_ipython():
+ # default shell should be 'ipython'
+ shell = get_shell_embed_func()
+ assert shell.__name__ == "_embed_ipython_shell"
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index e8dd88049..b1532ca77 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -1,4 +1,5 @@
import warnings
+from typing import Any
import pytest
from w3lib.http import basic_auth_header
@@ -8,9 +9,8 @@ from scrapy.utils.curl import curl_to_request_kwargs
class TestCurlToRequestKwargs:
- maxDiff = 5000
-
- def _test_command(self, curl_command, expected_result):
+ @staticmethod
+ def _test_command(curl_command: str, expected_result: dict[str, Any]) -> None:
result = curl_to_request_kwargs(curl_command)
assert result == expected_result
try:
@@ -220,7 +220,7 @@ class TestCurlToRequestKwargs:
assert curl_to_request_kwargs(curl_command) == expected_result
# case 2: ignore_unknown_options=False (raise exception):
- with pytest.raises(ValueError, match="Unrecognized options:.*--bar.*--baz"):
+ with pytest.raises(ValueError, match=r"Unrecognized options:.*--bar.*--baz"):
curl_to_request_kwargs(
"curl --bar --baz http://www.example.com", ignore_unknown_options=False
)
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 75b6b0e99..c58b02ca5 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,5 +1,6 @@
import copy
import warnings
+from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping, MutableMapping
import pytest
@@ -16,7 +17,12 @@ from scrapy.utils.datatypes import (
from scrapy.utils.python import garbage_collect
-class CaseInsensitiveDictBase:
+class TestCaseInsensitiveDictBase(ABC):
+ @property
+ @abstractmethod
+ def dict_class(self) -> type[MutableMapping]:
+ raise NotImplementedError
+
def test_init_dict(self):
seq = {"red": 1, "black": 3}
d = self.dict_class(seq)
@@ -199,7 +205,7 @@ class CaseInsensitiveDictBase:
assert h1.get("header1") == h3.get("HEADER1")
-class TestCaseInsensitiveDict(CaseInsensitiveDictBase):
+class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
dict_class = CaseInsensitiveDict
def test_repr(self):
@@ -216,7 +222,7 @@ class TestCaseInsensitiveDict(CaseInsensitiveDictBase):
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestCaselessDict(CaseInsensitiveDictBase):
+class TestCaselessDict(TestCaseInsensitiveDictBase):
dict_class = CaselessDict
def test_deprecation_message(self):
@@ -347,26 +353,26 @@ class TestLocalWeakReferencedCache:
assert len(cache) == 0
def test_cache_without_limit(self):
- max = 10**4
+ maximum = 10**4
cache = LocalWeakReferencedCache()
refs = []
- for x in range(max):
+ for x in range(maximum):
refs.append(Request(f"https://example.org/{x}"))
cache[refs[-1]] = x
- assert len(cache) == max
+ assert len(cache) == maximum
for i, r in enumerate(refs):
assert r in cache
assert cache[r] == i
del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable
# delete half of the objects, make sure that is reflected in the cache
- for _ in range(max // 2):
+ for _ in range(maximum // 2):
refs.pop()
# PyPy takes longer to collect dead references
garbage_collect()
- assert len(cache) == max // 2
+ assert len(cache) == maximum // 2
for i, r in enumerate(refs):
assert r in cache
assert cache[r] == i
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 1cfaf70fa..df7f1a20e 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -7,8 +7,6 @@ from typing import TYPE_CHECKING, Any
import pytest
from twisted.internet.defer import Deferred, inlineCallbacks, succeed
-from twisted.python.failure import Failure
-from twisted.trial import unittest
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import (
@@ -20,28 +18,32 @@ from scrapy.utils.defer import (
maybe_deferred_to_future,
mustbe_deferred,
parallel_async,
- process_chain,
- process_parallel,
)
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
-class TestMustbeDeferred(unittest.TestCase):
- def test_success_function(self) -> Deferred[list[int]]:
+@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
+class TestMustbeDeferred:
+ @inlineCallbacks
+ def test_success_function(self) -> Generator[Deferred[Any], Any, None]:
steps: list[int] = []
def _append(v: int) -> list[int]:
steps.append(v)
return steps
- dfd = mustbe_deferred(_append, 1)
- dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
- steps.append(2) # add another value, that should be caught by assertEqual
- return dfd
+ def _assert(v: list[int]) -> None:
+ assert v == [1, 2] # it is [1] with maybeDeferred
- def test_unfired_deferred(self) -> Deferred[list[int]]:
+ dfd = mustbe_deferred(_append, 1)
+ dfd.addCallback(_assert)
+ steps.append(2) # add another value, that should be caught by assertEqual
+ yield dfd
+
+ @inlineCallbacks
+ def test_unfired_deferred(self) -> Generator[Deferred[Any], Any, None]:
steps: list[int] = []
def _append(v: int) -> Deferred[list[int]]:
@@ -52,10 +54,13 @@ class TestMustbeDeferred(unittest.TestCase):
reactor.callLater(0, dfd.callback, steps)
return dfd
+ def _assert(v: list[int]) -> None:
+ assert v == [1, 2]
+
dfd = mustbe_deferred(_append, 1)
- dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
+ dfd.addCallback(_assert)
steps.append(2) # add another value, that should be caught by assertEqual
- return dfd
+ yield dfd
def cb1(value, arg1, arg2):
@@ -71,33 +76,13 @@ def cb3(value, arg1, arg2):
def cb_fail(value, arg1, arg2):
- return Failure(TypeError())
+ raise TypeError
def eb1(failure, arg1, arg2):
return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})"
-class TestDeferUtils(unittest.TestCase):
- @inlineCallbacks
- def test_process_chain(self):
- x = yield process_chain([cb1, cb2, cb3], "res", "v1", "v2")
- assert x == "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)"
-
- with pytest.raises(TypeError):
- yield process_chain([cb1, cb_fail, cb3], "res", "v1", "v2")
-
- @inlineCallbacks
- def test_process_parallel(self):
- x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2")
- assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"]
-
- @inlineCallbacks
- def test_process_parallel_failure(self):
- with pytest.raises(TypeError):
- yield process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2")
-
-
class TestIterErrback:
def test_iter_errback_good(self):
def itergood() -> Generator[int, None, None]:
@@ -122,7 +107,7 @@ class TestIterErrback:
assert isinstance(errors[0].value, ZeroDivisionError)
-class TestAiterErrback(unittest.TestCase):
+class TestAiterErrback:
@deferred_f_from_coro_f
async def test_aiter_errback_good(self):
async def itergood() -> AsyncGenerator[int, None]:
@@ -149,7 +134,7 @@ class TestAiterErrback(unittest.TestCase):
assert isinstance(errors[0].value, ZeroDivisionError)
-class TestAsyncDefTestsuite(unittest.TestCase):
+class TestAsyncDefTestsuite:
@deferred_f_from_coro_f
async def test_deferred_f_from_coro_f(self):
pass
@@ -164,7 +149,7 @@ class TestAsyncDefTestsuite(unittest.TestCase):
raise RuntimeError("This is expected to be raised")
-class TestParallelAsync(unittest.TestCase):
+class TestParallelAsync:
"""This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.
parallel_async is called with the results of a callback (so an iterable of items, requests and None,
@@ -274,7 +259,7 @@ class TestParallelAsync(unittest.TestCase):
assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0]
-class TestDeferredFromCoro(unittest.TestCase):
+class TestDeferredFromCoro:
def test_deferred(self):
d = Deferred()
result = deferred_from_coro(d)
@@ -318,7 +303,7 @@ class TestDeferredFromCoro(unittest.TestCase):
assert future_result == 42
-class TestDeferredFFromCoroF(unittest.TestCase):
+class TestDeferredFFromCoroF:
@inlineCallbacks
def _assert_result(
self, c_f: Callable[[], Awaitable[int]]
@@ -355,7 +340,7 @@ class TestDeferredFFromCoroF(unittest.TestCase):
@pytest.mark.only_asyncio
-class TestDeferredToFuture(unittest.TestCase):
+class TestDeferredToFuture:
@deferred_f_from_coro_f
async def test_deferred(self):
d = Deferred()
@@ -390,7 +375,7 @@ class TestDeferredToFuture(unittest.TestCase):
@pytest.mark.only_asyncio
-class TestMaybeDeferredToFutureAsyncio(unittest.TestCase):
+class TestMaybeDeferredToFutureAsyncio:
@deferred_f_from_coro_f
async def test_deferred(self):
d = Deferred()
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 662de0dc3..a88b5e008 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -1,6 +1,7 @@
import inspect
import warnings
from unittest import mock
+from warnings import WarningMessage
import pytest
@@ -21,7 +22,9 @@ class NewName(SomeBaseClass):
class TestWarnWhenSubclassed:
- def _mywarnings(self, w, category=MyWarning):
+ def _mywarnings(
+ self, w: list[WarningMessage], category: type[Warning] = MyWarning
+ ) -> list[WarningMessage]:
return [x for x in w if x.category is MyWarning]
def test_no_warning_on_definition(self):
diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
index cea564653..8b02116b8 100644
--- a/tests/test_utils_display.py
+++ b/tests/test_utils_display.py
@@ -1,90 +1,93 @@
+import builtins
from io import StringIO
from unittest import mock
from scrapy.utils.display import pformat, pprint
-
-class TestDisplay:
- object = {"a": 1}
- colorized_strings = {
+value = {"a": 1}
+colorized_strings = {
+ (
(
- (
- "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
- "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}"
- )
- + suffix
+ "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
+ "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}"
)
- for suffix in (
- # https://github.com/pygments/pygments/issues/2313
- "\n", # pygments ≤ 2.13
- "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14
- )
- }
- plain_string = "{'a': 1}"
+ + suffix
+ )
+ for suffix in (
+ # https://github.com/pygments/pygments/issues/2313
+ "\n", # pygments ≤ 2.13
+ "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14
+ )
+}
+plain_string = "{'a': 1}"
- @mock.patch("sys.platform", "linux")
- @mock.patch("sys.stdout.isatty")
- def test_pformat(self, isatty):
- isatty.return_value = True
- assert pformat(self.object) in self.colorized_strings
- @mock.patch("sys.stdout.isatty")
- def test_pformat_dont_colorize(self, isatty):
- isatty.return_value = True
- assert pformat(self.object, colorize=False) == self.plain_string
+@mock.patch("sys.platform", "linux")
+@mock.patch("sys.stdout.isatty")
+def test_pformat(isatty):
+ isatty.return_value = True
+ assert pformat(value) in colorized_strings
- def test_pformat_not_tty(self):
- assert pformat(self.object) == self.plain_string
- @mock.patch("sys.platform", "win32")
- @mock.patch("platform.version")
- @mock.patch("sys.stdout.isatty")
- def test_pformat_old_windows(self, isatty, version):
- isatty.return_value = True
- version.return_value = "10.0.14392"
- assert pformat(self.object) in self.colorized_strings
+@mock.patch("sys.stdout.isatty")
+def test_pformat_dont_colorize(isatty):
+ isatty.return_value = True
+ assert pformat(value, colorize=False) == plain_string
- @mock.patch("sys.platform", "win32")
- @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
- @mock.patch("platform.version")
- @mock.patch("sys.stdout.isatty")
- def test_pformat_windows_no_terminal_processing(
- self, isatty, version, terminal_processing
- ):
- isatty.return_value = True
- version.return_value = "10.0.14393"
- terminal_processing.return_value = False
- assert pformat(self.object) == self.plain_string
- @mock.patch("sys.platform", "win32")
- @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
- @mock.patch("platform.version")
- @mock.patch("sys.stdout.isatty")
- def test_pformat_windows(self, isatty, version, terminal_processing):
- isatty.return_value = True
- version.return_value = "10.0.14393"
- terminal_processing.return_value = True
- assert pformat(self.object) in self.colorized_strings
+def test_pformat_not_tty():
+ assert pformat(value) == plain_string
- @mock.patch("sys.platform", "linux")
- @mock.patch("sys.stdout.isatty")
- def test_pformat_no_pygments(self, isatty):
- isatty.return_value = True
- import builtins
+@mock.patch("sys.platform", "win32")
+@mock.patch("platform.version")
+@mock.patch("sys.stdout.isatty")
+def test_pformat_old_windows(isatty, version):
+ isatty.return_value = True
+ version.return_value = "10.0.14392"
+ assert pformat(value) in colorized_strings
- real_import = builtins.__import__
- def mock_import(name, globals, locals, fromlist, level):
- if "pygments" in name:
- raise ImportError
- return real_import(name, globals, locals, fromlist, level)
+@mock.patch("sys.platform", "win32")
+@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
+@mock.patch("platform.version")
+@mock.patch("sys.stdout.isatty")
+def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing):
+ isatty.return_value = True
+ version.return_value = "10.0.14393"
+ terminal_processing.return_value = False
+ assert pformat(value) == plain_string
- builtins.__import__ = mock_import
- assert pformat(self.object) == self.plain_string
- builtins.__import__ = real_import
- def test_pprint(self):
- with mock.patch("sys.stdout", new=StringIO()) as mock_out:
- pprint(self.object)
- assert mock_out.getvalue() == "{'a': 1}\n"
+@mock.patch("sys.platform", "win32")
+@mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
+@mock.patch("platform.version")
+@mock.patch("sys.stdout.isatty")
+def test_pformat_windows(isatty, version, terminal_processing):
+ isatty.return_value = True
+ version.return_value = "10.0.14393"
+ terminal_processing.return_value = True
+ assert pformat(value) in colorized_strings
+
+
+@mock.patch("sys.platform", "linux")
+@mock.patch("sys.stdout.isatty")
+def test_pformat_no_pygments(isatty):
+ isatty.return_value = True
+
+ real_import = builtins.__import__
+
+ def mock_import(name, globals_, locals_, fromlist, level):
+ if "pygments" in name:
+ raise ImportError
+ return real_import(name, globals_, locals_, fromlist, level)
+
+ builtins.__import__ = mock_import
+ assert pformat(value) == plain_string
+ builtins.__import__ = real_import
+
+
+def test_pprint():
+ with mock.patch("sys.stdout", new=StringIO()) as mock_out:
+ pprint(value)
+ assert mock_out.getvalue() == "{'a': 1}\n"
diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index c43ed152b..06fdf9cba 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -11,47 +11,51 @@ from tests import tests_datadir
SAMPLEDIR = Path(tests_datadir, "compressed")
-class TestGunzip:
- def test_gunzip_basic(self):
- r1 = Response(
- "http://www.example.com",
- body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(),
- )
- assert gzip_magic_number(r1)
+def test_gunzip_basic():
+ r1 = Response(
+ "http://www.example.com",
+ body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(),
+ )
+ assert gzip_magic_number(r1)
- r2 = Response("http://www.example.com", body=gunzip(r1.body))
- assert not gzip_magic_number(r2)
- assert len(r2.body) == 9950
+ r2 = Response("http://www.example.com", body=gunzip(r1.body))
+ assert not gzip_magic_number(r2)
+ assert len(r2.body) == 9950
- def test_gunzip_truncated(self):
- text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes())
- assert text.endswith(b"")
- assert not gzip_magic_number(r2)
+def test_gunzip_no_gzip_file_raises():
+ with pytest.raises(BadGzipFile):
+ gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes())
- def test_is_gzipped_empty(self):
- r1 = Response("http://www.example.com")
- assert not gzip_magic_number(r1)
- def test_gunzip_illegal_eof(self):
- text = html_to_unicode(
- "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes())
- )[1]
- expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text(
- encoding="utf-8"
- )
- assert len(text) == len(expected_text)
- assert text == expected_text
+def test_gunzip_truncated_short():
+ r1 = Response(
+ "http://www.example.com",
+ body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(),
+ )
+ assert gzip_magic_number(r1)
+
+ r2 = Response("http://www.example.com", body=gunzip(r1.body))
+ assert r2.body.endswith(b"")
+ assert not gzip_magic_number(r2)
+
+
+def test_is_gzipped_empty():
+ r1 = Response("http://www.example.com")
+ assert not gzip_magic_number(r1)
+
+
+def test_gunzip_illegal_eof():
+ text = html_to_unicode(
+ "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes())
+ )[1]
+ expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text(
+ encoding="utf-8"
+ )
+ assert len(text) == len(expected_text)
+ assert text == expected_text
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index 0c05ef7d6..9bd86f7fb 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -4,18 +4,17 @@ from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
-class TestHttpobjUtils:
- def test_urlparse_cached(self):
- url = "http://www.example.com/index.html"
- request1 = Request(url)
- request2 = Request(url)
- req1a = urlparse_cached(request1)
- req1b = urlparse_cached(request1)
- req2 = urlparse_cached(request2)
- urlp = urlparse(url)
+def test_urlparse_cached():
+ url = "http://www.example.com/index.html"
+ request1 = Request(url)
+ request2 = Request(url)
+ req1a = urlparse_cached(request1)
+ req1b = urlparse_cached(request1)
+ req2 = urlparse_cached(request2)
+ urlp = urlparse(url)
- assert req1a == req2
- assert req1a == urlp
- assert req1a is req1b
- assert req1a is not req2
- assert req1a is not req2
+ assert req1a == req2
+ assert req1a == urlp
+ assert req1a is req1b
+ assert req1a is not req2
+ assert req1a is not req2
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index fa0d37866..73e55b736 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,3 +1,8 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -5,9 +10,19 @@ from scrapy.http import Response, TextResponse, XmlResponse
from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml
from tests import get_testdata
+if TYPE_CHECKING:
+ from collections.abc import Iterator
+
+ from scrapy import Selector
+
+
+class TestXmliterBase(ABC):
+ @abstractmethod
+ def xmliter(
+ self, obj: Response | str | bytes, nodename: str, *args: Any
+ ) -> Iterator[Selector]:
+ raise NotImplementedError
-class XmliterBase:
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter(self):
body = b"""
@@ -39,7 +54,6 @@ class XmliterBase:
("002", ["Name 2"], ["Type 2"]),
]
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_unusual_node(self):
body = b"""
@@ -53,7 +67,6 @@ class XmliterBase:
]
assert nodenames == [["matchme..."]]
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_unicode(self):
# example taken from https://github.com/scrapy/scrapy/issues/1665
body = """
@@ -113,7 +126,6 @@ class XmliterBase:
("27", ["A"], ["27"]),
]
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_text(self):
body = (
''
@@ -125,7 +137,6 @@ class XmliterBase:
["two"],
]
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_namespaces(self):
body = b"""
@@ -163,7 +174,6 @@ class XmliterBase:
assert node.xpath("id/text()").getall() == []
assert node.xpath("price/text()").getall() == []
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_namespaced_nodename(self):
body = b"""
@@ -191,7 +201,6 @@ class XmliterBase:
"http://www.mydummycompany.com/images/item1.jpg"
]
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_namespaced_nodename_missing(self):
body = b"""
@@ -216,26 +225,23 @@ class XmliterBase:
with pytest.raises(StopIteration):
next(my_iter)
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_exception(self):
body = (
''
"one two ]+>$")
+
+ @pytest.fixture
+ def log_stream(self) -> StringIO:
+ return StringIO()
+
+ @pytest.fixture
+ def spider(self) -> LogSpider:
+ return LogSpider()
+
+ @pytest.fixture(autouse=True)
+ def logger(self, log_stream: StringIO) -> Generator[logging.Logger]:
+ handler = logging.StreamHandler(log_stream)
formatter = logging.Formatter(
'{"levelname": "%(levelname)s", "message": "%(message)s", "spider": "%(spider)s", "important_info": "%(important_info)s"}'
)
@@ -209,80 +235,79 @@ class TestLoggingWithExtra:
logger = logging.getLogger("log_spider")
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
- self.handler = handler
- self.logger = logger
- self.spider = LogSpider()
- self.regex_pattern = re.compile(r"^]+>$")
- def teardown_method(self):
- self.logger.removeHandler(self.handler)
+ yield logger
- def test_debug_logging(self):
+ logger.removeHandler(handler)
+
+ def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo message"
extra = {"important_info": "foo"}
- self.spider.log_debug(log_message, extra)
- log_contents = self.log_stream.getvalue()
- log_contents = json.loads(log_contents)
+ spider.log_debug(log_message, extra)
+ log_contents_str = log_stream.getvalue()
+ log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "DEBUG"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
- def test_info_logging(self):
+ def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Bar message"
extra = {"important_info": "bar"}
- self.spider.log_info(log_message, extra)
- log_contents = self.log_stream.getvalue()
- log_contents = json.loads(log_contents)
+ spider.log_info(log_message, extra)
+ log_contents_str = log_stream.getvalue()
+ log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "INFO"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
- def test_warning_logging(self):
+ def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Baz message"
extra = {"important_info": "baz"}
- self.spider.log_warning(log_message, extra)
- log_contents = self.log_stream.getvalue()
- log_contents = json.loads(log_contents)
+ spider.log_warning(log_message, extra)
+ log_contents_str = log_stream.getvalue()
+ log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "WARNING"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
- def test_error_logging(self):
+ def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo bar message"
extra = {"important_info": "foo bar"}
- self.spider.log_error(log_message, extra)
- log_contents = self.log_stream.getvalue()
- log_contents = json.loads(log_contents)
+ spider.log_error(log_message, extra)
+ log_contents_str = log_stream.getvalue()
+ log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "ERROR"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
- def test_critical_logging(self):
+ def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo bar baz message"
extra = {"important_info": "foo bar baz"}
- self.spider.log_critical(log_message, extra)
- log_contents = self.log_stream.getvalue()
- log_contents = json.loads(log_contents)
+ spider.log_critical(log_message, extra)
+ log_contents_str = log_stream.getvalue()
+ log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "CRITICAL"
assert log_contents["message"] == log_message
assert self.regex_pattern.match(log_contents["spider"])
assert log_contents["important_info"] == extra["important_info"]
- def test_overwrite_spider_extra(self):
+ def test_overwrite_spider_extra(
+ self, log_stream: StringIO, spider: LogSpider
+ ) -> None:
log_message = "Foo message"
extra = {"important_info": "foo", "spider": "shouldn't change"}
- self.spider.log_error(log_message, extra)
- log_contents = self.log_stream.getvalue()
- log_contents = json.loads(log_contents)
+ spider.log_error(log_message, extra)
+ log_contents_str = log_stream.getvalue()
+ log_contents = json.loads(log_contents_str)
assert log_contents["levelname"] == "ERROR"
assert log_contents["message"] == log_message
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index b330819d9..610fc0ffe 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -9,7 +9,6 @@ from scrapy.item import Field, Item
from scrapy.utils.misc import (
arg_to_iter,
build_from_crawler,
- create_instance,
load_object,
rel_has_nofollow,
set_environ,
@@ -97,98 +96,29 @@ class TestUtilsMisc:
assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_create_instance(self):
- settings = mock.MagicMock()
- crawler = mock.MagicMock(spec_set=["settings"])
- args = (True, 100.0)
- kwargs = {"key": "val"}
-
- def _test_with_settings(mock, settings):
- create_instance(mock, settings, None, *args, **kwargs)
- if hasattr(mock, "from_crawler"):
- assert mock.from_crawler.call_count == 0
- if hasattr(mock, "from_settings"):
- mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
- assert mock.call_count == 0
- else:
- mock.assert_called_once_with(*args, **kwargs)
-
- def _test_with_crawler(mock, settings, crawler):
- create_instance(mock, settings, crawler, *args, **kwargs)
- if hasattr(mock, "from_crawler"):
- mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
- if hasattr(mock, "from_settings"):
- assert mock.from_settings.call_count == 0
- assert mock.call_count == 0
- elif hasattr(mock, "from_settings"):
- mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
- assert mock.call_count == 0
- else:
- mock.assert_called_once_with(*args, **kwargs)
-
- # Check usage of correct constructor using four mocks:
- # 1. with no alternative constructors
- # 2. with from_settings() constructor
- # 3. with from_crawler() constructor
- # 4. with from_settings() and from_crawler() constructor
- spec_sets = (
- ["__qualname__"],
- ["__qualname__", "from_settings"],
- ["__qualname__", "from_crawler"],
- ["__qualname__", "from_settings", "from_crawler"],
- )
- for specs in spec_sets:
- m = mock.MagicMock(spec_set=specs)
- _test_with_settings(m, settings)
- m.reset_mock()
- _test_with_crawler(m, settings, crawler)
-
- # Check adoption of crawler settings
- m = mock.MagicMock(spec_set=["__qualname__", "from_settings"])
- create_instance(m, None, crawler, *args, **kwargs)
- m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs)
-
- with pytest.raises(
- ValueError, match="Specify at least one of settings and crawler"
- ):
- create_instance(m, None, None)
-
- m.from_settings.return_value = None
- with pytest.raises(TypeError):
- create_instance(m, settings, None)
-
def test_build_from_crawler(self):
- settings = mock.MagicMock()
crawler = mock.MagicMock(spec_set=["settings"])
args = (True, 100.0)
kwargs = {"key": "val"}
- def _test_with_crawler(mock, settings, crawler):
+ def _test_with_crawler(mock, crawler):
build_from_crawler(mock, crawler, *args, **kwargs)
if hasattr(mock, "from_crawler"):
mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
- if hasattr(mock, "from_settings"):
- assert mock.from_settings.call_count == 0
- assert mock.call_count == 0
- elif hasattr(mock, "from_settings"):
- mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
assert mock.call_count == 0
else:
mock.assert_called_once_with(*args, **kwargs)
- # Check usage of correct constructor using three mocks:
+ # Check usage of correct constructor using 2 mocks:
# 1. with no alternative constructors
# 2. with from_crawler() constructor
- # 3. with from_settings() and from_crawler() constructor
spec_sets = (
["__qualname__"],
["__qualname__", "from_crawler"],
- ["__qualname__", "from_settings", "from_crawler"],
)
for specs in spec_sets:
m = mock.MagicMock(spec_set=specs)
- _test_with_crawler(m, settings, crawler)
+ _test_with_crawler(m, crawler)
m.reset_mock()
# Check adoption of crawler
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index aa250be69..20a3d940c 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -1,18 +1,17 @@
-import contextlib
import os
-import shutil
-import tempfile
import warnings
from pathlib import Path
+import pytest
+
from scrapy.utils.misc import set_environ
from scrapy.utils.project import data_path, get_project_settings
-@contextlib.contextmanager
-def inside_a_project():
+@pytest.fixture
+def proj_path(tmp_path):
prev_dir = Path.cwd()
- project_dir = tempfile.mkdtemp()
+ project_dir = tmp_path
try:
os.chdir(project_dir)
@@ -21,21 +20,19 @@ def inside_a_project():
yield project_dir
finally:
os.chdir(prev_dir)
- shutil.rmtree(project_dir)
-class TestProjectUtils:
- def test_data_path_outside_project(self):
- assert str(Path(".scrapy", "somepath")) == data_path("somepath")
- abspath = str(Path(os.path.sep, "absolute", "path"))
- assert abspath == data_path(abspath)
+def test_data_path_outside_project():
+ assert str(Path(".scrapy", "somepath")) == data_path("somepath")
+ abspath = str(Path(os.path.sep, "absolute", "path"))
+ assert abspath == data_path(abspath)
- def test_data_path_inside_project(self):
- with inside_a_project() as proj_path:
- expected = Path(proj_path, ".scrapy", "somepath")
- assert expected.resolve() == Path(data_path("somepath")).resolve()
- abspath = str(Path(os.path.sep, "absolute", "path").resolve())
- assert abspath == data_path(abspath)
+
+def test_data_path_inside_project(proj_path: Path) -> None:
+ expected = proj_path / ".scrapy" / "somepath"
+ assert expected.resolve() == Path(data_path("somepath")).resolve()
+ abspath = str(Path(os.path.sep, "absolute", "path").resolve())
+ assert abspath == data_path(abspath)
class TestGetProjectSettings:
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 291646ad7..34fca06c1 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -1,10 +1,12 @@
+from __future__ import annotations
+
import functools
import operator
import platform
import sys
+from typing import TYPE_CHECKING, TypeVar
import pytest
-from twisted.trial import unittest
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import aiter_errback, deferred_f_from_coro_f
@@ -12,7 +14,6 @@ from scrapy.utils.python import (
MutableAsyncChain,
MutableChain,
binary_is_text,
- equal_attributes,
get_func_args,
memoizemethod_noargs,
to_bytes,
@@ -20,19 +21,25 @@ from scrapy.utils.python import (
without_none_values,
)
-
-class TestMutableChain:
- def test_mutablechain(self):
- m = MutableChain(range(2), [2, 3], (4, 5))
- m.extend(range(6, 7))
- m.extend([7, 8])
- m.extend([9, 10], (11, 12))
- assert next(m) == 0
- assert m.__next__() == 1
- assert list(m) == list(range(2, 13))
+if TYPE_CHECKING:
+ from collections.abc import Iterable, Mapping
-class TestMutableAsyncChain(unittest.TestCase):
+_KT = TypeVar("_KT")
+_VT = TypeVar("_VT")
+
+
+def test_mutablechain():
+ m = MutableChain(range(2), [2, 3], (4, 5))
+ m.extend(range(6, 7))
+ m.extend([7, 8])
+ m.extend([9, 10], (11, 12))
+ assert next(m) == 0
+ assert m.__next__() == 1
+ assert list(m) == list(range(2, 13))
+
+
+class TestMutableAsyncChain:
@staticmethod
async def g1():
for i in range(3):
@@ -112,144 +119,100 @@ class TestToBytes:
assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b"
-class TestMemoizedMethod:
- def test_memoizemethod_noargs(self):
- class A:
- @memoizemethod_noargs
- def cached(self):
- return object()
+def test_memoizemethod_noargs():
+ class A:
+ @memoizemethod_noargs
+ def cached(self):
+ return object()
- def noncached(self):
- return object()
+ def noncached(self):
+ return object()
- a = A()
- one = a.cached()
- two = a.cached()
- three = a.noncached()
- assert one is two
- assert one is not three
+ a = A()
+ one = a.cached()
+ two = a.cached()
+ three = a.noncached()
+ assert one is two
+ assert one is not three
-class TestBinaryIsText:
- def test_binaryistext(self):
- assert binary_is_text(b"hello")
-
- def test_utf_16_strings_contain_null_bytes(self):
- assert binary_is_text("hello".encode("utf-16"))
-
- def test_one_with_encoding(self):
- assert binary_is_text(b"Price \xa3
")
-
- def test_real_binary_bytes(self):
- assert not binary_is_text(b"\x02\xa3")
+@pytest.mark.parametrize(
+ ("value", "expected"),
+ [
+ (b"hello", True),
+ ("hello".encode("utf-16"), True),
+ (b"Price \xa3
", True),
+ (b"\x02\xa3", False),
+ ],
+)
+def test_binaryistext(value: bytes, expected: bool) -> None:
+ assert binary_is_text(value) is expected
-class TestUtilsPython:
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_equal_attributes(self):
- class Obj:
+def test_get_func_args():
+ def f1(a, b, c):
+ pass
+
+ def f2(a, b=None, c=None):
+ pass
+
+ def f3(a, b=None, *, c=None):
+ pass
+
+ class A:
+ def __init__(self, a, b, c):
pass
- a = Obj()
- b = Obj()
- # no attributes given return False
- assert not equal_attributes(a, b, [])
- # nonexistent attributes
- assert not equal_attributes(a, b, ["x", "y"])
-
- a.x = 1
- b.x = 1
- # equal attribute
- assert equal_attributes(a, b, ["x"])
-
- b.y = 2
- # obj1 has no attribute y
- assert not equal_attributes(a, b, ["x", "y"])
-
- a.y = 2
- # equal attributes
- assert equal_attributes(a, b, ["x", "y"])
-
- a.y = 1
- # different attributes
- assert not equal_attributes(a, b, ["x", "y"])
-
- # test callable
- a.meta = {}
- b.meta = {}
- assert equal_attributes(a, b, ["meta"])
-
- # compare ['meta']['a']
- a.meta["z"] = 1
- b.meta["z"] = 1
-
- get_z = operator.itemgetter("z")
- get_meta = operator.attrgetter("meta")
-
- def compare_z(obj):
- return get_z(get_meta(obj))
-
- assert equal_attributes(a, b, [compare_z, "x"])
- # fail z equality
- a.meta["z"] = 2
- assert not equal_attributes(a, b, [compare_z, "x"])
-
- def test_get_func_args(self):
- def f1(a, b, c):
+ def method(self, a, b, c):
pass
- def f2(a, b=None, c=None):
+ class Callable:
+ def __call__(self, a, b, c):
pass
- def f3(a, b=None, *, c=None):
- pass
+ a = A(1, 2, 3)
+ cal = Callable()
+ partial_f1 = functools.partial(f1, None)
+ partial_f2 = functools.partial(f1, b=None)
+ partial_f3 = functools.partial(partial_f2, None)
- class A:
- def __init__(self, a, b, c):
- pass
+ assert get_func_args(f1) == ["a", "b", "c"]
+ assert get_func_args(f2) == ["a", "b", "c"]
+ assert get_func_args(f3) == ["a", "b", "c"]
+ assert get_func_args(A) == ["a", "b", "c"]
+ assert get_func_args(a.method) == ["a", "b", "c"]
+ assert get_func_args(partial_f1) == ["b", "c"]
+ assert get_func_args(partial_f2) == ["a", "c"]
+ assert get_func_args(partial_f3) == ["c"]
+ assert get_func_args(cal) == ["a", "b", "c"]
+ assert get_func_args(object) == [] # pylint: disable=use-implicit-booleaness-not-comparison
+ assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"]
+ assert get_func_args(" ".join, stripself=True) == ["iterable"]
- def method(self, a, b, c):
- pass
+ if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy":
+ # the correct and correctly extracted signature
+ assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"]
+ elif platform.python_implementation() == "CPython":
+ # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature
+ # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951)
+ assert get_func_args(operator.itemgetter(2), stripself=True) in [
+ [],
+ ["args", "kwargs"],
+ ]
- class Callable:
- def __call__(self, a, b, c):
- pass
- a = A(1, 2, 3)
- cal = Callable()
- partial_f1 = functools.partial(f1, None)
- partial_f2 = functools.partial(f1, b=None)
- partial_f3 = functools.partial(partial_f2, None)
-
- assert get_func_args(f1) == ["a", "b", "c"]
- assert get_func_args(f2) == ["a", "b", "c"]
- assert get_func_args(f3) == ["a", "b", "c"]
- assert get_func_args(A) == ["a", "b", "c"]
- assert get_func_args(a.method) == ["a", "b", "c"]
- assert get_func_args(partial_f1) == ["b", "c"]
- assert get_func_args(partial_f2) == ["a", "c"]
- assert get_func_args(partial_f3) == ["c"]
- assert get_func_args(cal) == ["a", "b", "c"]
- assert get_func_args(object) == []
- assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"]
- assert get_func_args(" ".join, stripself=True) == ["iterable"]
-
- if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy":
- # the correct and correctly extracted signature
- assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"]
- elif platform.python_implementation() == "CPython":
- # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature
- # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951)
- assert get_func_args(operator.itemgetter(2), stripself=True) in [
- [],
- ["args", "kwargs"],
- ]
-
- def test_without_none_values(self):
- assert without_none_values([1, None, 3, 4]) == [1, 3, 4]
- assert without_none_values((1, None, 3, 4)) == (1, 3, 4)
- assert without_none_values({"one": 1, "none": None, "three": 3, "four": 4}) == {
- "one": 1,
- "three": 3,
- "four": 4,
- }
+@pytest.mark.parametrize(
+ ("value", "expected"),
+ [
+ ([1, None, 3, 4], [1, 3, 4]),
+ ((1, None, 3, 4), (1, 3, 4)),
+ (
+ {"one": 1, "none": None, "three": 3, "four": 4},
+ {"one": 1, "three": 3, "four": 4},
+ ),
+ ],
+)
+def test_without_none_values(
+ value: Mapping[_KT, _VT] | Iterable[_KT], expected: dict[_KT, _VT] | Iterable[_KT]
+) -> None:
+ assert without_none_values(value) == expected
diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py
index eb00ab193..6cbdcfccc 100644
--- a/tests/test_utils_reactor.py
+++ b/tests/test_utils_reactor.py
@@ -2,7 +2,6 @@ import asyncio
import warnings
import pytest
-from twisted.trial.unittest import TestCase
from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.reactor import (
@@ -13,11 +12,10 @@ from scrapy.utils.reactor import (
)
-@pytest.mark.usefixtures("reactor_pytest")
-class TestAsyncio(TestCase):
- def test_is_asyncio_reactor_installed(self):
+class TestAsyncio:
+ def test_is_asyncio_reactor_installed(self, reactor_pytest: str) -> None:
# the result should depend only on the pytest --reactor argument
- assert is_asyncio_reactor_installed() == (self.reactor_pytest != "default")
+ assert is_asyncio_reactor_installed() == (reactor_pytest == "asyncio")
def test_install_asyncio_reactor(self):
from twisted.internet import reactor as original_reactor
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 5b8509753..b25129c87 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,64 +1,61 @@
from __future__ import annotations
import json
-import warnings
from hashlib import sha1
from weakref import WeakKeyDictionary
import pytest
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.utils.python import to_bytes
from scrapy.utils.request import (
_fingerprint_cache,
fingerprint,
- request_authenticate,
request_httprepr,
request_to_curl,
)
from scrapy.utils.test import get_crawler
-class TestUtilsRequest:
- @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
- def test_request_authenticate(self):
- r = Request("http://www.example.com")
- request_authenticate(r, "someuser", "somepass")
- assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M="
+@pytest.mark.parametrize(
+ ("r", "expected"),
+ [
+ (
+ Request("http://www.example.com"),
+ b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
+ ),
+ (
+ Request("http://www.example.com/some/page.html?arg=1"),
+ b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
+ ),
+ (
+ Request(
+ "http://www.example.com",
+ method="POST",
+ headers={"Content-type": b"text/html"},
+ body=b"Some body",
+ ),
+ b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body",
+ ),
+ ],
+)
+def test_request_httprepr(r: Request, expected: bytes) -> None:
+ assert request_httprepr(r) == expected
- def test_request_httprepr(self):
- r1 = Request("http://www.example.com")
- assert (
- request_httprepr(r1) == b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"
- )
- r1 = Request("http://www.example.com/some/page.html?arg=1")
- assert (
- request_httprepr(r1)
- == b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n"
- )
-
- r1 = Request(
- "http://www.example.com",
- method="POST",
- headers={"Content-type": b"text/html"},
- body=b"Some body",
- )
- assert (
- request_httprepr(r1)
- == b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body"
- )
-
- def test_request_httprepr_for_non_http_request(self):
- # the representation is not important but it must not fail.
- request_httprepr(Request("file:///tmp/foo.txt"))
- request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
+@pytest.mark.parametrize(
+ "r",
+ [
+ Request("file:///tmp/foo.txt"),
+ Request("ftp://localhost/tmp/foo.txt"),
+ ],
+)
+def test_request_httprepr_for_non_http_request(r: Request) -> None:
+ # the representation is not important but it must not fail.
+ request_httprepr(r)
class TestFingerprint:
- maxDiff = None
-
function: staticmethod = staticmethod(fingerprint)
cache: (
WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]]
@@ -229,55 +226,14 @@ class TestFingerprint:
assert actual == expected
-REQUEST_OBJECTS_TO_TEST = (
- Request("http://www.example.com/"),
- Request("http://www.example.com/query?id=111&cat=222"),
- Request("http://www.example.com/query?cat=222&id=111"),
- Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"),
- Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"),
- Request("http://www.example.com/members/offers.html"),
- Request(
- "http://www.example.com/members/offers.html",
- headers={"SESSIONID": b"somehash"},
- ),
- Request(
- "http://www.example.com/",
- headers={"Accept-Language": b"en"},
- ),
- Request(
- "http://www.example.com/",
- headers={
- "Accept-Language": b"en",
- "SESSIONID": b"somehash",
- },
- ),
- Request("http://www.example.com/test.html"),
- Request("http://www.example.com/test.html#fragment"),
- Request("http://www.example.com", method="POST"),
- Request("http://www.example.com", method="POST", body=b"request body"),
-)
-
-
class TestRequestFingerprinter:
- def test_default_implementation(self):
+ def test_fingerprint(self):
crawler = get_crawler()
request = Request("https://example.com")
assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
request
)
- def test_deprecated_implementation(self):
- settings = {
- "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
- }
- with warnings.catch_warnings(record=True) as logged_warnings:
- crawler = get_crawler(settings_dict=settings)
- request = Request("https://example.com")
- assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
- request
- )
- assert logged_warnings
-
class TestCustomRequestFingerprinter:
def test_include_headers(self):
@@ -365,57 +321,6 @@ class TestCustomRequestFingerprinter:
fingerprint = crawler.request_fingerprinter.fingerprint(request)
assert fingerprint == settings["FINGERPRINT"]
- def test_from_settings(self):
- class RequestFingerprinter:
- @classmethod
- def from_settings(cls, settings):
- return cls(settings)
-
- def __init__(self, settings):
- self._fingerprint = settings["FINGERPRINT"]
-
- def fingerprint(self, request):
- return self._fingerprint
-
- settings = {
- "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
- "FINGERPRINT": b"fingerprint",
- }
- with warnings.catch_warnings():
- warnings.simplefilter("ignore", ScrapyDeprecationWarning)
- crawler = get_crawler(settings_dict=settings)
-
- request = Request("http://www.example.com")
- fingerprint = crawler.request_fingerprinter.fingerprint(request)
- assert fingerprint == settings["FINGERPRINT"]
-
- def test_from_crawler_and_settings(self):
- class RequestFingerprinter:
- # This method is ignored due to the presence of from_crawler
- @classmethod
- def from_settings(cls, settings):
- return cls(settings)
-
- @classmethod
- def from_crawler(cls, crawler):
- return cls(crawler)
-
- def __init__(self, crawler):
- self._fingerprint = crawler.settings["FINGERPRINT"]
-
- def fingerprint(self, request):
- return self._fingerprint
-
- settings = {
- "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
- "FINGERPRINT": b"fingerprint",
- }
- crawler = get_crawler(settings_dict=settings)
-
- request = Request("http://www.example.com")
- fingerprint = crawler.request_fingerprinter.fingerprint(request)
- assert fingerprint == settings["FINGERPRINT"]
-
class TestRequestToCurl:
def _test_request(self, request_object, expected_curl_command):
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 80f2f25d5..0aeb5594f 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -4,7 +4,7 @@ from urllib.parse import urlparse
import pytest
-from scrapy.http import HtmlResponse, Response, TextResponse
+from scrapy.http import HtmlResponse, Response
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (
_remove_html_comments,
@@ -15,229 +15,217 @@ from scrapy.utils.response import (
)
-class TestResponseUtils:
- dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response")
+def test_open_in_browser():
+ url = "http:///www.example.com/some/page.html"
+ body = (
+ b" test page test body "
+ )
- def test_open_in_browser(self):
- url = "http:///www.example.com/some/page.html"
- body = b" test page test body "
+ def browser_open(burl: str) -> bool:
+ path = urlparse(burl).path
+ if not path or not Path(path).exists():
+ path = burl.replace("file://", "")
+ bbody = Path(path).read_bytes()
+ assert b'Dummy Dummy
-
- blahablsdfsal&
- """,
- )
- r3 = HtmlResponse(
- "http://www.example.com",
- body=b"""
- Dummy Dummy
+
+ blahablsdfsal&
+ """,
+ )
+ r3 = HtmlResponse(
+ "http://www.example.com",
+ body=b"""
+Dummy
+ Dummy
- Hello world.
- """,
- )
- r2 = HtmlResponse(
- url,
- body=b"""
-
- Dummy
- Hello world.
- """,
- )
- r3 = HtmlResponse(
- url,
- body=b"""
-
- Dummy
-
-
- Hello world.
-
- """,
- )
- r4 = HtmlResponse(
- url,
- body=b"""
-
-
- Dummy
- Hello world.
- """,
- )
- r5 = HtmlResponse(
- url,
- body=b"""
-
-
-
- Standard head
-
- Hello world.
- """,
- )
- assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
- assert open_in_browser(r2, _openfunc=check_base_url), (
- "Inject base url with argumented head"
- )
- assert open_in_browser(r3, _openfunc=check_base_url), (
- "Inject unique base url with misleading tag"
- )
- assert open_in_browser(r4, _openfunc=check_base_url), (
- "Inject unique base url with misleading comment"
- )
- assert open_in_browser(r5, _openfunc=check_base_url), (
- "Inject unique base url with conditional comment"
- )
+def test_inject_base_url():
+ url = "http://www.example.com"
- def test_open_in_browser_redos_comment(self):
- MAX_CPU_TIME = 0.02
+ def check_base_url(burl):
+ path = urlparse(burl).path
+ if not path or not Path(path).exists():
+ path = burl.replace("file://", "")
+ bbody = Path(path).read_bytes()
+ assert bbody.count(b'Dummy
+ Hello world.
+ """,
+ )
+ r2 = HtmlResponse(
+ url,
+ body=b"""
+
+ Dummy
+ Hello world.
+ """,
+ )
+ r3 = HtmlResponse(
+ url,
+ body=b"""
+
+ Dummy
+
+
+ Hello world.
+
+ """,
+ )
+ r4 = HtmlResponse(
+ url,
+ body=b"""
+
+
+ Dummy
+ Hello world.
+ """,
+ )
+ r5 = HtmlResponse(
+ url,
+ body=b"""
+
+
+
+ Standard head
+
+ Hello world.
+ """,
+ )
- response = HtmlResponse("https://example.com", body=body)
+ assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
+ assert open_in_browser(r2, _openfunc=check_base_url), (
+ "Inject base url with argumented head"
+ )
+ assert open_in_browser(r3, _openfunc=check_base_url), (
+ "Inject unique base url with misleading tag"
+ )
+ assert open_in_browser(r4, _openfunc=check_base_url), (
+ "Inject unique base url with misleading comment"
+ )
+ assert open_in_browser(r5, _openfunc=check_base_url), (
+ "Inject unique base url with conditional comment"
+ )
- start_time = process_time()
- open_in_browser(response, lambda url: True)
+def test_open_in_browser_redos_comment():
+ MAX_CPU_TIME = 0.02
- end_time = process_time()
- assert end_time - start_time < MAX_CPU_TIME
+ # Exploit input from
+ # https://makenowjust-labs.github.io/recheck/playground/
+ # for // (old pattern to remove comments).
+ body = b"->"
+ response = HtmlResponse("https://example.com", body=body)
+ start_time = process_time()
+ open_in_browser(response, lambda url: True)
+ end_time = process_time()
+ assert end_time - start_time < MAX_CPU_TIME
- def test_open_in_browser_redos_head(self):
- MAX_CPU_TIME = 0.02
- # Exploit input from
- # https://makenowjust-labs.github.io/recheck/playground/
- # for /(|\s.*?>))/ (old pattern to find the head element).
- body = b"|\s.*?>))/ (old pattern to find the head element).
+ body = b"b",
- b"ab",
- ),
- (
- b"ac",
- b"ac",
- ),
- (
- b"acccd",
- b"acd",
- ),
- (
- b"ad",
- b"ad",
- ),
+ (b"ab", b"ab"),
+ (b"ac", b"ac"),
+ (b"acccd", b"acd"),
+ (b"ad", b"ad"),
],
)
def test_remove_html_comments(input_body, output_body):
- assert _remove_html_comments(input_body) == output_body, (
- f"{_remove_html_comments(input_body)=} == {output_body=}"
- )
+ assert _remove_html_comments(input_body) == output_body
diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py
index 2ee3850b0..2e6a790f8 100644
--- a/tests/test_utils_serialize.py
+++ b/tests/test_utils_serialize.py
@@ -4,6 +4,7 @@ import json
from decimal import Decimal
import attr
+import pytest
from twisted.internet import defer
from scrapy.http import Request, Response
@@ -11,10 +12,11 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
class TestJsonEncoder:
- def setup_method(self):
- self.encoder = ScrapyJSONEncoder(sort_keys=True)
+ @pytest.fixture
+ def encoder(self) -> ScrapyJSONEncoder:
+ return ScrapyJSONEncoder(sort_keys=True)
- def test_encode_decode(self):
+ def test_encode_decode(self, encoder: ScrapyJSONEncoder) -> None:
dt = datetime.datetime(2010, 1, 2, 10, 11, 12)
dts = "2010-01-02 10:11:12"
d = datetime.date(2010, 1, 2)
@@ -28,7 +30,7 @@ class TestJsonEncoder:
dt_set = {dt}
dt_sets = [dts]
- for input, output in [
+ for input_, output in [
("foo", "foo"),
(d, ds),
(t, ts),
@@ -38,24 +40,24 @@ class TestJsonEncoder:
(s, ss),
(dt_set, dt_sets),
]:
- assert self.encoder.encode(input) == json.dumps(output, sort_keys=True)
+ assert encoder.encode(input_) == json.dumps(output, sort_keys=True)
- def test_encode_deferred(self):
- assert "Deferred" in self.encoder.encode(defer.Deferred())
+ def test_encode_deferred(self, encoder: ScrapyJSONEncoder) -> None:
+ assert "Deferred" in encoder.encode(defer.Deferred())
- def test_encode_request(self):
+ def test_encode_request(self, encoder: ScrapyJSONEncoder) -> None:
r = Request("http://www.example.com/lala")
- rs = self.encoder.encode(r)
+ rs = encoder.encode(r)
assert r.method in rs
assert r.url in rs
- def test_encode_response(self):
+ def test_encode_response(self, encoder: ScrapyJSONEncoder) -> None:
r = Response("http://www.example.com/lala")
- rs = self.encoder.encode(r)
+ rs = encoder.encode(r)
assert r.url in rs
assert str(r.status) in rs
- def test_encode_dataclass_item(self) -> None:
+ def test_encode_dataclass_item(self, encoder: ScrapyJSONEncoder) -> None:
@dataclasses.dataclass
class TestDataClass:
name: str
@@ -63,10 +65,10 @@ class TestJsonEncoder:
price: int
item = TestDataClass(name="Product", url="http://product.org", price=1)
- encoded = self.encoder.encode(item)
+ encoded = encoder.encode(item)
assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}'
- def test_encode_attrs_item(self):
+ def test_encode_attrs_item(self, encoder: ScrapyJSONEncoder) -> None:
@attr.s
class AttrsItem:
name = attr.ib(type=str)
@@ -74,5 +76,5 @@ class TestJsonEncoder:
price = attr.ib(type=int)
item = AttrsItem(name="Product", url="http://product.org", price=1)
- encoded = self.encoder.encode(item)
+ encoded = encoder.encode(item)
assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}'
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index 79bac8bc5..6e4bdb49a 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -6,8 +6,8 @@ from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.python.failure import Failure
-from twisted.trial import unittest
+from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.signal import (
send_catch_log,
@@ -17,7 +17,7 @@ from scrapy.utils.signal import (
from scrapy.utils.test import get_from_asyncio_queue
-class TestSendCatchLog(unittest.TestCase):
+class TestSendCatchLog:
@inlineCallbacks
def test_send_catch_log(self):
test_signal = object()
@@ -66,16 +66,13 @@ class TestSendCatchLogDeferred(TestSendCatchLog):
class TestSendCatchLogDeferred2(TestSendCatchLogDeferred):
def ok_handler(self, arg, handlers_called):
- from twisted.internet import reactor
-
handlers_called.add(self.ok_handler)
assert arg == "test"
d = defer.Deferred()
- reactor.callLater(0, d.callback, "OK")
+ call_later(0, d.callback, "OK")
return d
-@pytest.mark.usefixtures("reactor_pytest")
class TestSendCatchLogDeferredAsyncDef(TestSendCatchLogDeferred):
async def ok_handler(self, arg, handlers_called):
handlers_called.add(self.ok_handler)
@@ -100,16 +97,13 @@ class TestSendCatchLogAsync(TestSendCatchLog):
class TestSendCatchLogAsync2(TestSendCatchLogAsync):
def ok_handler(self, arg, handlers_called):
- from twisted.internet import reactor
-
handlers_called.add(self.ok_handler)
assert arg == "test"
d = defer.Deferred()
- reactor.callLater(0, d.callback, "OK")
+ call_later(0, d.callback, "OK")
return d
-@pytest.mark.usefixtures("reactor_pytest")
class TestSendCatchLogAsyncAsyncDef(TestSendCatchLogAsync):
async def ok_handler(self, arg, handlers_called):
handlers_called.add(self.ok_handler)
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index 36d612009..464a31777 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -1,158 +1,162 @@
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
-class TestSitemap:
- def test_sitemap(self):
- s = Sitemap(
- b"""
+def test_sitemap():
+ s = Sitemap(
+ b"""
-
- http://www.example.com/
- 2009-08-16
- daily
- 1
-
-
- http://www.example.com/Special-Offers.html
- 2009-08-16
- weekly
- 0.8
-
+
+http://www.example.com/
+2009-08-16
+daily
+1
+
+
+http://www.example.com/Special-Offers.html
+2009-08-16
+weekly
+0.8
+
"""
- )
- assert s.type == "urlset"
- assert list(s) == [
- {
- "priority": "1",
- "loc": "http://www.example.com/",
- "lastmod": "2009-08-16",
- "changefreq": "daily",
- },
- {
- "priority": "0.8",
- "loc": "http://www.example.com/Special-Offers.html",
- "lastmod": "2009-08-16",
- "changefreq": "weekly",
- },
- ]
+ )
+ assert s.type == "urlset"
+ assert list(s) == [
+ {
+ "priority": "1",
+ "loc": "http://www.example.com/",
+ "lastmod": "2009-08-16",
+ "changefreq": "daily",
+ },
+ {
+ "priority": "0.8",
+ "loc": "http://www.example.com/Special-Offers.html",
+ "lastmod": "2009-08-16",
+ "changefreq": "weekly",
+ },
+ ]
- def test_sitemap_index(self):
- s = Sitemap(
- b"""
+
+def test_sitemap_index():
+ s = Sitemap(
+ b"""
-
- http://www.example.com/sitemap1.xml.gz
- 2004-10-01T18:23:17+00:00
-
-
- http://www.example.com/sitemap2.xml.gz
- 2005-01-01
-
+
+ http://www.example.com/sitemap1.xml.gz
+ 2004-10-01T18:23:17+00:00
+
+
+ http://www.example.com/sitemap2.xml.gz
+ 2005-01-01
+
"""
- )
- assert s.type == "sitemapindex"
- assert list(s) == [
- {
- "loc": "http://www.example.com/sitemap1.xml.gz",
- "lastmod": "2004-10-01T18:23:17+00:00",
- },
- {
- "loc": "http://www.example.com/sitemap2.xml.gz",
- "lastmod": "2005-01-01",
- },
- ]
+ )
+ assert s.type == "sitemapindex"
+ assert list(s) == [
+ {
+ "loc": "http://www.example.com/sitemap1.xml.gz",
+ "lastmod": "2004-10-01T18:23:17+00:00",
+ },
+ {
+ "loc": "http://www.example.com/sitemap2.xml.gz",
+ "lastmod": "2005-01-01",
+ },
+ ]
- def test_sitemap_strip(self):
- """Assert we can deal with trailing spaces inside tags - we've
- seen those
- """
- s = Sitemap(
- b"""
+
+def test_sitemap_strip():
+ """Assert we can deal with trailing spaces inside tags - we've
+ seen those
+ """
+ s = Sitemap(
+ b"""
-
- http://www.example.com/
- 2009-08-16
- daily
- 1
-
-
- http://www.example.com/2
-
+
+ http://www.example.com/
+2009-08-16
+daily
+1
+
+
+ http://www.example.com/2
+
"""
- )
- assert list(s) == [
- {
- "priority": "1",
- "loc": "http://www.example.com/",
- "lastmod": "2009-08-16",
- "changefreq": "daily",
- },
- {"loc": "http://www.example.com/2", "lastmod": ""},
- ]
+ )
+ assert list(s) == [
+ {
+ "priority": "1",
+ "loc": "http://www.example.com/",
+ "lastmod": "2009-08-16",
+ "changefreq": "daily",
+ },
+ {"loc": "http://www.example.com/2", "lastmod": ""},
+ ]
- def test_sitemap_wrong_ns(self):
- """We have seen sitemaps with wrongs ns. Presumably, Google still works
- with these, though is not 100% confirmed"""
- s = Sitemap(
- b"""
+
+def test_sitemap_wrong_ns():
+ """We have seen sitemaps with wrongs ns. Presumably, Google still works
+ with these, though is not 100% confirmed"""
+ s = Sitemap(
+ b"""
-
- http://www.example.com/
- 2009-08-16
- daily
- 1
-
-
- http://www.example.com/2
-
+
+ http://www.example.com/
+2009-08-16
+daily
+1
+
+
+ http://www.example.com/2
+
"""
- )
- assert list(s) == [
- {
- "priority": "1",
- "loc": "http://www.example.com/",
- "lastmod": "2009-08-16",
- "changefreq": "daily",
- },
- {"loc": "http://www.example.com/2", "lastmod": ""},
- ]
+ )
+ assert list(s) == [
+ {
+ "priority": "1",
+ "loc": "http://www.example.com/",
+ "lastmod": "2009-08-16",
+ "changefreq": "daily",
+ },
+ {"loc": "http://www.example.com/2", "lastmod": ""},
+ ]
- def test_sitemap_wrong_ns2(self):
- """We have seen sitemaps with wrongs ns. Presumably, Google still works
- with these, though is not 100% confirmed"""
- s = Sitemap(
- b"""
+
+def test_sitemap_wrong_ns2():
+ """We have seen sitemaps with wrongs ns. Presumably, Google still works
+ with these, though is not 100% confirmed"""
+ s = Sitemap(
+ b"""
-
- http://www.example.com/
- 2009-08-16
- daily
- 1
-
-
- http://www.example.com/2
-
+
+ http://www.example.com/
+2009-08-16
+daily
+1
+
+
+ http://www.example.com/2
+
"""
- )
- assert s.type == "urlset"
- assert list(s) == [
- {
- "priority": "1",
- "loc": "http://www.example.com/",
- "lastmod": "2009-08-16",
- "changefreq": "daily",
- },
- {"loc": "http://www.example.com/2", "lastmod": ""},
- ]
+ )
+ assert s.type == "urlset"
+ assert list(s) == [
+ {
+ "priority": "1",
+ "loc": "http://www.example.com/",
+ "lastmod": "2009-08-16",
+ "changefreq": "daily",
+ },
+ {"loc": "http://www.example.com/2", "lastmod": ""},
+ ]
- def test_sitemap_urls_from_robots(self):
- robots = """User-agent: *
+
+def test_sitemap_urls_from_robots():
+ robots = """User-agent: *
Disallow: /aff/
Disallow: /wl/
@@ -170,19 +174,18 @@ Sitemap: /sitemap-relative-url.xml
Disallow: /forum/search/
Disallow: /forum/active/
"""
- assert list(
- sitemap_urls_from_robots(robots, base_url="http://example.com")
- ) == [
- "http://example.com/sitemap.xml",
- "http://example.com/sitemap-product-index.xml",
- "http://example.com/sitemap-uppercase.xml",
- "http://example.com/sitemap-relative-url.xml",
- ]
+ assert list(sitemap_urls_from_robots(robots, base_url="http://example.com")) == [
+ "http://example.com/sitemap.xml",
+ "http://example.com/sitemap-product-index.xml",
+ "http://example.com/sitemap-uppercase.xml",
+ "http://example.com/sitemap-relative-url.xml",
+ ]
- def test_sitemap_blanklines(self):
- """Assert we can deal with starting blank lines before tag"""
- s = Sitemap(
- b"""
+
+def test_sitemap_blanklines():
+ """Assert we can deal with starting blank lines before tag"""
+ s = Sitemap(
+ b"""
@@ -205,69 +208,69 @@ Disallow: /forum/active/
"""
- )
- assert list(s) == [
- {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap1.xml"},
- {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap2.xml"},
- {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"},
- ]
+ )
+ assert list(s) == [
+ {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap1.xml"},
+ {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap2.xml"},
+ {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"},
+ ]
- def test_comment(self):
- s = Sitemap(
- b"""
-
+
+def test_comment():
+ s = Sitemap(
+ b"""
+
+
+ http://www.example.com/
+
+
+ """
+ )
+ assert list(s) == [{"loc": "http://www.example.com/"}]
+
+
+def test_alternate():
+ s = Sitemap(
+ b"""
+
+
+ http://www.example.com/english/
+
+ """
+ )
+ assert list(s) == [
+ {
+ "loc": "http://www.example.com/english/",
+ "alternate": [
+ "http://www.example.com/deutsch/",
+ "http://www.example.com/schweiz-deutsch/",
+ "http://www.example.com/english/",
+ ],
+ }
+ ]
+
+
+def test_xml_entity_expansion():
+ s = Sitemap(
+ b"""
+
+
+ ]>
+
- http://www.example.com/
-
+ http://127.0.0.1:8000/&xxe;
- """
- )
-
- assert list(s) == [{"loc": "http://www.example.com/"}]
-
- def test_alternate(self):
- s = Sitemap(
- b"""
-
-
- http://www.example.com/english/
-
- """
- )
-
- assert list(s) == [
- {
- "loc": "http://www.example.com/english/",
- "alternate": [
- "http://www.example.com/deutsch/",
- "http://www.example.com/schweiz-deutsch/",
- "http://www.example.com/english/",
- ],
- }
- ]
-
- def test_xml_entity_expansion(self):
- s = Sitemap(
- b"""
-
-
- ]>
-
-
- http://127.0.0.1:8000/&xxe;
-
-
- """
- )
-
- assert list(s) == [{"loc": "http://127.0.0.1:8000/"}]
+
+ """
+ )
+ assert list(s) == [{"loc": "http://127.0.0.1:8000/"}]
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index 43e603f6c..5efb29a49 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -12,19 +12,19 @@ class MySpider2(Spider):
name = "myspider2"
-class TestUtilsSpiders:
- def test_iterate_spider_output(self):
- i = Item()
- r = Request("http://scrapytest.org")
- o = object()
+def test_iterate_spider_output():
+ i = Item()
+ r = Request("http://scrapytest.org")
+ o = object()
- assert list(iterate_spider_output(i)) == [i]
- assert list(iterate_spider_output(r)) == [r]
- assert list(iterate_spider_output(o)) == [o]
- assert list(iterate_spider_output([r, i, o])) == [r, i, o]
+ assert list(iterate_spider_output(i)) == [i]
+ assert list(iterate_spider_output(r)) == [r]
+ assert list(iterate_spider_output(o)) == [o]
+ assert list(iterate_spider_output([r, i, o])) == [r, i, o]
- def test_iter_spider_classes(self):
- import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self
- it = iter_spider_classes(tests.test_utils_spider)
- assert set(it) == {MySpider1, MySpider2}
+def test_iter_spider_classes():
+ import tests.test_utils_spider # noqa: PLW0406,PLC0415 # pylint: disable=import-self
+
+ it = iter_spider_classes(tests.test_utils_spider)
+ assert set(it) == {MySpider1, MySpider2}
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 41d9b8933..4515ce36e 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -1,22 +1,21 @@
from scrapy.utils.template import render_templatefile
-class TestUtilsRenderTemplateFile:
- def test_simple_render(self, tmp_path):
- context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}
- template = "from ${project_name}.spiders.${name} import ${classname}"
- rendered = "from proj.spiders.spi import TheSpider"
+def test_simple_render(tmp_path):
+ context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}
+ template = "from ${project_name}.spiders.${name} import ${classname}"
+ rendered = "from proj.spiders.spi import TheSpider"
- template_path = tmp_path / "templ.py.tmpl"
- render_path = tmp_path / "templ.py"
+ template_path = tmp_path / "templ.py.tmpl"
+ render_path = tmp_path / "templ.py"
- template_path.write_text(template, encoding="utf8")
- assert template_path.is_file() # Failure of test itself
+ template_path.write_text(template, encoding="utf8")
+ assert template_path.is_file() # Failure of test itself
- render_templatefile(template_path, **context)
+ render_templatefile(template_path, **context)
- assert not template_path.exists()
- assert render_path.read_text(encoding="utf8") == rendered
+ assert not template_path.exists()
+ assert render_path.read_text(encoding="utf8") == rendered
- render_path.unlink()
- assert not render_path.exists() # Failure of test itself
+ render_path.unlink()
+ assert not render_path.exists() # Failure of test itself
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index e3d6675bf..3967c3365 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -15,71 +15,76 @@ class Bar(trackref.object_ref):
pass
-class TestTrackref:
- def setup_method(self):
- trackref.live_refs.clear()
+@pytest.fixture(autouse=True)
+def clear_refs() -> None:
+ trackref.live_refs.clear()
- def test_format_live_refs(self):
- o1 = Foo() # noqa: F841
- o2 = Bar() # noqa: F841
- o3 = Foo() # noqa: F841
- assert (
- trackref.format_live_refs()
- == """\
+
+def test_format_live_refs():
+ o1 = Foo() # noqa: F841
+ o2 = Bar() # noqa: F841
+ o3 = Foo() # noqa: F841
+ assert (
+ trackref.format_live_refs()
+ == """\
Live References
Bar 1 oldest: 0s ago
Foo 2 oldest: 0s ago
"""
- )
+ )
- assert (
- trackref.format_live_refs(ignore=Foo)
- == """\
+ assert (
+ trackref.format_live_refs(ignore=Foo)
+ == """\
Live References
Bar 1 oldest: 0s ago
"""
- )
+ )
- @mock.patch("sys.stdout", new_callable=StringIO)
- def test_print_live_refs_empty(self, stdout):
- trackref.print_live_refs()
- assert stdout.getvalue() == "Live References\n\n\n"
- @mock.patch("sys.stdout", new_callable=StringIO)
- def test_print_live_refs_with_objects(self, stdout):
- o1 = Foo() # noqa: F841
- trackref.print_live_refs()
- assert (
- stdout.getvalue()
- == """\
+@mock.patch("sys.stdout", new_callable=StringIO)
+def test_print_live_refs_empty(stdout):
+ trackref.print_live_refs()
+ assert stdout.getvalue() == "Live References\n\n\n"
+
+
+@mock.patch("sys.stdout", new_callable=StringIO)
+def test_print_live_refs_with_objects(stdout):
+ o1 = Foo() # noqa: F841
+ trackref.print_live_refs()
+ assert (
+ stdout.getvalue()
+ == """\
Live References
Foo 1 oldest: 0s ago\n\n"""
- )
+ )
- def test_get_oldest(self):
- o1 = Foo()
- o1_time = time()
+def test_get_oldest():
+ o1 = Foo()
- o2 = Bar()
+ o1_time = time()
+ o2 = Bar()
+
+ o3_time = time()
+ if o3_time <= o1_time:
+ sleep(0.01)
o3_time = time()
- if o3_time <= o1_time:
- sleep(0.01)
- o3_time = time()
- if o3_time <= o1_time:
- pytest.skip("time.time is not precise enough")
+ if o3_time <= o1_time:
+ pytest.skip("time.time is not precise enough")
- o3 = Foo() # noqa: F841
- assert trackref.get_oldest("Foo") is o1
- assert trackref.get_oldest("Bar") is o2
- assert trackref.get_oldest("XXX") is None
+ o3 = Foo() # noqa: F841
+ assert trackref.get_oldest("Foo") is o1
+ assert trackref.get_oldest("Bar") is o2
+ assert trackref.get_oldest("XXX") is None
- def test_iter_all(self):
- o1 = Foo()
- o2 = Bar() # noqa: F841
- o3 = Foo()
- assert set(trackref.iter_all("Foo")) == {o1, o3}
+
+def test_iter_all():
+ o1 = Foo()
+ o2 = Bar() # noqa: F841
+ o3 = Foo()
+ assert set(trackref.iter_all("Foo")) == {o1, o3}
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index c85dcc55e..bcbbe74a3 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,10 +1,10 @@
import warnings
+from importlib import import_module
import pytest
from scrapy.linkextractors import IGNORED_EXTENSIONS
from scrapy.spiders import Spider
-from scrapy.utils.misc import arg_to_iter
from scrapy.utils.url import ( # type: ignore[attr-defined]
_is_filesystem_path,
_public_w3lib_objects,
@@ -17,261 +17,156 @@ from scrapy.utils.url import ( # type: ignore[attr-defined]
)
-class TestUrlUtils:
- def test_url_is_from_any_domain(self):
- url = "http://www.wheele-bin-art.co.uk/get/product/123"
- assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
- assert not url_is_from_any_domain(url, ["art.co.uk"])
+def test_url_is_from_any_domain():
+ url = "http://www.wheele-bin-art.co.uk/get/product/123"
+ assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
+ assert not url_is_from_any_domain(url, ["art.co.uk"])
- url = "http://wheele-bin-art.co.uk/get/product/123"
- assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
- assert not url_is_from_any_domain(url, ["art.co.uk"])
+ url = "http://wheele-bin-art.co.uk/get/product/123"
+ assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
+ assert not url_is_from_any_domain(url, ["art.co.uk"])
- url = "http://www.Wheele-Bin-Art.co.uk/get/product/123"
- assert url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"])
- assert url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"])
+ url = "http://www.Wheele-Bin-Art.co.uk/get/product/123"
+ assert url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"])
+ assert url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"])
- url = "http://192.169.0.15:8080/mypage.html"
- assert url_is_from_any_domain(url, ["192.169.0.15:8080"])
- assert not url_is_from_any_domain(url, ["192.169.0.15"])
+ url = "http://192.169.0.15:8080/mypage.html"
+ assert url_is_from_any_domain(url, ["192.169.0.15:8080"])
+ assert not url_is_from_any_domain(url, ["192.169.0.15"])
- url = (
- "javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20"
- "javascript:%20document.orderform_2581_1190810811.submit%28%29"
- )
- assert not url_is_from_any_domain(url, ["testdomain.com"])
- assert not url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"])
-
- def test_url_is_from_spider(self):
- spider = Spider(name="example.com")
- assert url_is_from_spider("http://www.example.com/some/page.html", spider)
- assert url_is_from_spider("http://sub.example.com/some/page.html", spider)
- assert not url_is_from_spider("http://www.example.org/some/page.html", spider)
- assert not url_is_from_spider("http://www.example.net/some/page.html", spider)
-
- def test_url_is_from_spider_class_attributes(self):
- class MySpider(Spider):
- name = "example.com"
-
- assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
- assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
- assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
- assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
-
- def test_url_is_from_spider_with_allowed_domains(self):
- spider = Spider(
- name="example.com", allowed_domains=["example.org", "example.net"]
- )
- assert url_is_from_spider("http://www.example.com/some/page.html", spider)
- assert url_is_from_spider("http://sub.example.com/some/page.html", spider)
- assert url_is_from_spider("http://example.com/some/page.html", spider)
- assert url_is_from_spider("http://www.example.org/some/page.html", spider)
- assert url_is_from_spider("http://www.example.net/some/page.html", spider)
- assert not url_is_from_spider("http://www.example.us/some/page.html", spider)
-
- spider = Spider(
- name="example.com", allowed_domains={"example.com", "example.net"}
- )
- assert url_is_from_spider("http://www.example.com/some/page.html", spider)
-
- spider = Spider(
- name="example.com", allowed_domains=("example.com", "example.net")
- )
- assert url_is_from_spider("http://www.example.com/some/page.html", spider)
-
- def test_url_is_from_spider_with_allowed_domains_class_attributes(self):
- class MySpider(Spider):
- name = "example.com"
- allowed_domains = ("example.org", "example.net")
-
- assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
- assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
- assert url_is_from_spider("http://example.com/some/page.html", MySpider)
- assert url_is_from_spider("http://www.example.org/some/page.html", MySpider)
- assert url_is_from_spider("http://www.example.net/some/page.html", MySpider)
- assert not url_is_from_spider("http://www.example.us/some/page.html", MySpider)
-
- def test_url_has_any_extension(self):
- deny_extensions = {"." + e for e in arg_to_iter(IGNORED_EXTENSIONS)}
- assert url_has_any_extension(
- "http://www.example.com/archive.tar.gz", deny_extensions
- )
- assert url_has_any_extension("http://www.example.com/page.doc", deny_extensions)
- assert url_has_any_extension("http://www.example.com/page.pdf", deny_extensions)
- assert not url_has_any_extension(
- "http://www.example.com/page.htm", deny_extensions
- )
- assert not url_has_any_extension("http://www.example.com/", deny_extensions)
- assert not url_has_any_extension(
- "http://www.example.com/page.doc.html", deny_extensions
- )
+ url = (
+ "javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20"
+ "javascript:%20document.orderform_2581_1190810811.submit%28%29"
+ )
+ assert not url_is_from_any_domain(url, ["testdomain.com"])
+ assert not url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"])
-class TestAddHttpIfNoScheme:
- def test_add_scheme(self):
- assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"
+def test_url_is_from_spider():
+ class MySpider(Spider):
+ name = "example.com"
- def test_without_subdomain(self):
- assert add_http_if_no_scheme("example.com") == "http://example.com"
-
- def test_path(self):
- assert (
- add_http_if_no_scheme("www.example.com/some/page.html")
- == "http://www.example.com/some/page.html"
- )
-
- def test_port(self):
- assert (
- add_http_if_no_scheme("www.example.com:80") == "http://www.example.com:80"
- )
-
- def test_fragment(self):
- assert (
- add_http_if_no_scheme("www.example.com/some/page#frag")
- == "http://www.example.com/some/page#frag"
- )
-
- def test_query(self):
- assert (
- add_http_if_no_scheme("www.example.com/do?a=1&b=2&c=3")
- == "http://www.example.com/do?a=1&b=2&c=3"
- )
-
- def test_username_password(self):
- assert (
- add_http_if_no_scheme("username:password@www.example.com")
- == "http://username:password@www.example.com"
- )
-
- def test_complete_url(self):
- assert (
- add_http_if_no_scheme(
- "username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
- )
- == "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
- )
-
- def test_preserve_http(self):
- assert (
- add_http_if_no_scheme("http://www.example.com") == "http://www.example.com"
- )
-
- def test_preserve_http_without_subdomain(self):
- assert add_http_if_no_scheme("http://example.com") == "http://example.com"
-
- def test_preserve_http_path(self):
- assert (
- add_http_if_no_scheme("http://www.example.com/some/page.html")
- == "http://www.example.com/some/page.html"
- )
-
- def test_preserve_http_port(self):
- assert (
- add_http_if_no_scheme("http://www.example.com:80")
- == "http://www.example.com:80"
- )
-
- def test_preserve_http_fragment(self):
- assert (
- add_http_if_no_scheme("http://www.example.com/some/page#frag")
- == "http://www.example.com/some/page#frag"
- )
-
- def test_preserve_http_query(self):
- assert (
- add_http_if_no_scheme("http://www.example.com/do?a=1&b=2&c=3")
- == "http://www.example.com/do?a=1&b=2&c=3"
- )
-
- def test_preserve_http_username_password(self):
- assert (
- add_http_if_no_scheme("http://username:password@www.example.com")
- == "http://username:password@www.example.com"
- )
-
- def test_preserve_http_complete_url(self):
- assert (
- add_http_if_no_scheme(
- "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
- )
- == "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
- )
-
- def test_protocol_relative(self):
- assert add_http_if_no_scheme("//www.example.com") == "http://www.example.com"
-
- def test_protocol_relative_without_subdomain(self):
- assert add_http_if_no_scheme("//example.com") == "http://example.com"
-
- def test_protocol_relative_path(self):
- assert (
- add_http_if_no_scheme("//www.example.com/some/page.html")
- == "http://www.example.com/some/page.html"
- )
-
- def test_protocol_relative_port(self):
- assert (
- add_http_if_no_scheme("//www.example.com:80") == "http://www.example.com:80"
- )
-
- def test_protocol_relative_fragment(self):
- assert (
- add_http_if_no_scheme("//www.example.com/some/page#frag")
- == "http://www.example.com/some/page#frag"
- )
-
- def test_protocol_relative_query(self):
- assert (
- add_http_if_no_scheme("//www.example.com/do?a=1&b=2&c=3")
- == "http://www.example.com/do?a=1&b=2&c=3"
- )
-
- def test_protocol_relative_username_password(self):
- assert (
- add_http_if_no_scheme("//username:password@www.example.com")
- == "http://username:password@www.example.com"
- )
-
- def test_protocol_relative_complete_url(self):
- assert (
- add_http_if_no_scheme(
- "//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
- )
- == "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag"
- )
-
- def test_preserve_https(self):
- assert (
- add_http_if_no_scheme("https://www.example.com")
- == "https://www.example.com"
- )
-
- def test_preserve_ftp(self):
- assert add_http_if_no_scheme("ftp://www.example.com") == "ftp://www.example.com"
+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
+ assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
+ assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
+ assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
-class TestGuessScheme:
- pass
+def test_url_is_from_spider_class_attributes():
+ class MySpider(Spider):
+ name = "example.com"
+
+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
+ assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
+ assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
+ assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
-def create_guess_scheme_t(args):
- def do_expected(self):
- url = guess_scheme(args[0])
- assert url.startswith(args[1]), (
- f"Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`"
- )
+def test_url_is_from_spider_with_allowed_domains():
+ class MySpider(Spider):
+ name = "example.com"
+ allowed_domains = ["example.org", "example.net"]
- return do_expected
+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
+ assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
+ assert url_is_from_spider("http://example.com/some/page.html", MySpider)
+ assert url_is_from_spider("http://www.example.org/some/page.html", MySpider)
+ assert url_is_from_spider("http://www.example.net/some/page.html", MySpider)
+ assert not url_is_from_spider("http://www.example.us/some/page.html", MySpider)
+
+ class MySpider2(Spider):
+ name = "example.com"
+ allowed_domains = {"example.com", "example.net"}
+
+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider2)
+
+ class MySpider3(Spider):
+ name = "example.com"
+ allowed_domains = ("example.com", "example.net")
+
+ assert url_is_from_spider("http://www.example.com/some/page.html", MySpider3)
-def create_skipped_scheme_t(args):
- def do_expected(self):
- pytest.skip(args[2])
-
- return do_expected
+@pytest.mark.parametrize(
+ ("url", "expected"),
+ [
+ ("http://www.example.com/archive.tar.gz", True),
+ ("http://www.example.com/page.doc", True),
+ ("http://www.example.com/page.pdf", True),
+ ("http://www.example.com/page.htm", False),
+ ("http://www.example.com/", False),
+ ("http://www.example.com/page.doc.html", False),
+ ],
+)
+def test_url_has_any_extension(url: str, expected: bool) -> None:
+ deny_extensions = {"." + e for e in IGNORED_EXTENSIONS}
+ assert url_has_any_extension(url, deny_extensions) is expected
-for k, args in enumerate(
+@pytest.mark.parametrize(
+ ("url", "expected"),
+ [
+ ("www.example.com", "http://www.example.com"),
+ ("example.com", "http://example.com"),
+ ("www.example.com/some/page.html", "http://www.example.com/some/page.html"),
+ ("www.example.com:80", "http://www.example.com:80"),
+ ("www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),
+ ("www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),
+ (
+ "username:password@www.example.com",
+ "http://username:password@www.example.com",
+ ),
+ (
+ "username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
+ "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
+ ),
+ ("http://www.example.com", "http://www.example.com"),
+ ("http://example.com", "http://example.com"),
+ (
+ "http://www.example.com/some/page.html",
+ "http://www.example.com/some/page.html",
+ ),
+ ("http://www.example.com:80", "http://www.example.com:80"),
+ (
+ "http://www.example.com/some/page#frag",
+ "http://www.example.com/some/page#frag",
+ ),
+ (
+ "http://www.example.com/do?a=1&b=2&c=3",
+ "http://www.example.com/do?a=1&b=2&c=3",
+ ),
+ (
+ "http://username:password@www.example.com",
+ "http://username:password@www.example.com",
+ ),
+ (
+ "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
+ "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
+ ),
+ ("//www.example.com", "http://www.example.com"),
+ ("//example.com", "http://example.com"),
+ ("//www.example.com/some/page.html", "http://www.example.com/some/page.html"),
+ ("//www.example.com:80", "http://www.example.com:80"),
+ ("//www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),
+ ("//www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),
+ (
+ "//username:password@www.example.com",
+ "http://username:password@www.example.com",
+ ),
+ (
+ "//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
+ "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag",
+ ),
+ ("https://www.example.com", "https://www.example.com"),
+ ("ftp://www.example.com", "ftp://www.example.com"),
+ ],
+)
+def test_add_http_if_no_scheme(url: str, expected: str) -> None:
+ assert add_http_if_no_scheme(url) == expected
+
+
+@pytest.mark.parametrize(
+ ("url", "expected"),
[
("/index", "file://"),
("/index.html", "file://"),
@@ -295,14 +190,13 @@ for k, args in enumerate(
("/", "http://"),
(".../test", "http://"),
],
- start=1,
-):
- t_method = create_guess_scheme_t(args)
- t_method.__name__ = f"test_uri_{k:03}"
- setattr(TestGuessScheme, t_method.__name__, t_method)
+)
+def test_guess_scheme(url: str, expected: str):
+ assert guess_scheme(url).startswith(expected)
-# TODO: the following tests do not pass with current implementation
-for k, skip_args in enumerate(
+
+@pytest.mark.parametrize(
+ ("url", "expected", "reason"),
[
(
r"C:\absolute\path\to\a\file.html",
@@ -310,25 +204,21 @@ for k, skip_args in enumerate(
"Windows filepath are not supported for scrapy shell",
),
],
- start=1,
-):
- t_method = create_skipped_scheme_t(skip_args)
- t_method.__name__ = f"test_uri_skipped_{k:03}"
- setattr(TestGuessScheme, t_method.__name__, t_method)
+)
+def test_guess_scheme_skipped(url: str, expected: str, reason: str):
+ pytest.skip(reason)
class TestStripUrl:
- def test_noop(self):
- assert (
- strip_url("http://www.example.com/index.html")
- == "http://www.example.com/index.html"
- )
-
- def test_noop_query_string(self):
- assert (
- strip_url("http://www.example.com/index.html?somekey=somevalue")
- == "http://www.example.com/index.html?somekey=somevalue"
- )
+ @pytest.mark.parametrize(
+ "url",
+ [
+ "http://www.example.com/index.html",
+ "http://www.example.com/index.html?somekey=somevalue",
+ ],
+ )
+ def test_noop(self, url: str) -> None:
+ assert strip_url(url) == url
def test_fragments(self):
assert (
@@ -339,16 +229,20 @@ class TestStripUrl:
== "http://www.example.com/index.html?somekey=somevalue#section"
)
- def test_path(self):
- for input_url, origin, output_url in [
+ @pytest.mark.parametrize(
+ ("url", "origin", "expected"),
+ [
("http://www.example.com/", False, "http://www.example.com/"),
("http://www.example.com", False, "http://www.example.com"),
("http://www.example.com", True, "http://www.example.com/"),
- ]:
- assert strip_url(input_url, origin_only=origin) == output_url
+ ],
+ )
+ def test_path(self, url: str, origin: bool, expected: str) -> None:
+ assert strip_url(url, origin_only=origin) == expected
- def test_credentials(self):
- for i, o in [
+ @pytest.mark.parametrize(
+ ("url", "expected"),
+ [
(
"http://username@www.example.com/index.html?somekey=somevalue#section",
"http://www.example.com/index.html?somekey=somevalue",
@@ -361,34 +255,29 @@ class TestStripUrl:
"ftp://username:password@www.example.com/index.html?somekey=somevalue#section",
"ftp://www.example.com/index.html?somekey=somevalue",
),
- ]:
- assert strip_url(i, strip_credentials=True) == o
-
- def test_credentials_encoded_delims(self):
- for i, o in [
- # user: "username@"
- # password: none
+ # user: "username@", password: none
(
"http://username%40@www.example.com/index.html?somekey=somevalue#section",
"http://www.example.com/index.html?somekey=somevalue",
),
- # user: "username:pass"
- # password: ""
+ # user: "username:pass", password: ""
(
"https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section",
"https://www.example.com/index.html?somekey=somevalue",
),
- # user: "me"
- # password: "user@domain.com"
+ # user: "me", password: "user@domain.com"
(
"ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section",
"ftp://www.example.com/index.html?somekey=somevalue",
),
- ]:
- assert strip_url(i, strip_credentials=True) == o
+ ],
+ )
+ def test_credentials(self, url: str, expected: str) -> None:
+ assert strip_url(url, strip_credentials=True) == expected
- def test_default_ports_creds_off(self):
- for i, o in [
+ @pytest.mark.parametrize(
+ ("url", "expected"),
+ [
(
"http://username:password@www.example.com:80/index.html?somekey=somevalue#section",
"http://www.example.com/index.html?somekey=somevalue",
@@ -421,11 +310,14 @@ class TestStripUrl:
"ftp://username:password@www.example.com:221/file.txt",
"ftp://www.example.com:221/file.txt",
),
- ]:
- assert strip_url(i) == o
+ ],
+ )
+ def test_default_ports_creds_off(self, url: str, expected: str) -> None:
+ assert strip_url(url) == expected
- def test_default_ports(self):
- for i, o in [
+ @pytest.mark.parametrize(
+ ("url", "expected"),
+ [
(
"http://username:password@www.example.com:80/index.html",
"http://username:password@www.example.com/index.html",
@@ -458,11 +350,16 @@ class TestStripUrl:
"ftp://username:password@www.example.com:221/file.txt",
"ftp://username:password@www.example.com:221/file.txt",
),
- ]:
- assert strip_url(i, strip_default_port=True, strip_credentials=False) == o
+ ],
+ )
+ def test_default_ports(self, url: str, expected: str) -> None:
+ assert (
+ strip_url(url, strip_default_port=True, strip_credentials=False) == expected
+ )
- def test_default_ports_keep(self):
- for i, o in [
+ @pytest.mark.parametrize(
+ ("url", "expected"),
+ [
(
"http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section",
"http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov",
@@ -495,11 +392,17 @@ class TestStripUrl:
"ftp://username:password@www.example.com:221/file.txt",
"ftp://username:password@www.example.com:221/file.txt",
),
- ]:
- assert strip_url(i, strip_default_port=False, strip_credentials=False) == o
+ ],
+ )
+ def test_default_ports_keep(self, url: str, expected: str) -> None:
+ assert (
+ strip_url(url, strip_default_port=False, strip_credentials=False)
+ == expected
+ )
- def test_origin_only(self):
- for i, o in [
+ @pytest.mark.parametrize(
+ ("url", "expected"),
+ [
(
"http://username:password@www.example.com/index.html",
"http://www.example.com/",
@@ -516,29 +419,33 @@ class TestStripUrl:
"https://username:password@www.example.com:443/index.html",
"https://www.example.com/",
),
- ]:
- assert strip_url(i, origin_only=True) == o
+ ],
+ )
+ def test_origin_only(self, url: str, expected: str) -> None:
+ assert strip_url(url, origin_only=True) == expected
-class TestIsPath:
- def test_path(self):
- for input_value, output_value in (
- # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell
- # Unix-like OS, Microsoft Windows / cmd.exe
- ("/home/user/docs/Letter.txt", True),
- ("./inthisdir", True),
- ("../../greatgrandparent", True),
- ("~/.rcinfo", True),
- (r"C:\user\docs\Letter.txt", True),
- ("/user/docs/Letter.txt", True),
- (r"C:\Letter.txt", True),
- (r"\\Server01\user\docs\Letter.txt", True),
- (r"\\?\UNC\Server01\user\docs\Letter.txt", True),
- (r"\\?\C:\user\docs\Letter.txt", True),
- (r"C:\user\docs\somefile.ext:alternate_stream_name", True),
- (r"https://example.com", False),
- ):
- assert _is_filesystem_path(input_value) == output_value, input_value
+@pytest.mark.parametrize(
+ ("path", "expected"),
+ [
+ # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell
+ # Unix-like OS, Microsoft Windows / cmd.exe
+ ("/home/user/docs/Letter.txt", True),
+ ("./inthisdir", True),
+ ("../../greatgrandparent", True),
+ ("~/.rcinfo", True),
+ (r"C:\user\docs\Letter.txt", True),
+ ("/user/docs/Letter.txt", True),
+ (r"C:\Letter.txt", True),
+ (r"\\Server01\user\docs\Letter.txt", True),
+ (r"\\?\UNC\Server01\user\docs\Letter.txt", True),
+ (r"\\?\C:\user\docs\Letter.txt", True),
+ (r"C:\user\docs\somefile.ext:alternate_stream_name", True),
+ (r"https://example.com", False),
+ ],
+)
+def test__is_filesystem_path(path: str, expected: bool) -> None:
+ assert _is_filesystem_path(path) == expected
@pytest.mark.parametrize(
@@ -550,13 +457,12 @@ class TestIsPath:
*_public_w3lib_objects,
],
)
-def test_deprecated_imports_from_w3lib(obj_name):
+def test_deprecated_imports_from_w3lib(obj_name: str) -> None:
with warnings.catch_warnings(record=True) as warns:
obj_type = "attribute" if obj_name == "_safe_chars" else "function"
message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."
- from importlib import import_module
-
getattr(import_module("scrapy.utils.url"), obj_name)
+ assert isinstance(warns[0].message, Warning)
assert message in warns[0].message.args
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 569f4f639..963cb606e 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -1,41 +1,33 @@
"""
-from twisted.internet import defer
Tests borrowed from the twisted.web.client tests.
"""
from __future__ import annotations
-import shutil
-from pathlib import Path
-from tempfile import mkdtemp
from urllib.parse import urlparse
import OpenSSL.SSL
import pytest
+from pytest_twisted import async_yield_fixture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.internet.testing import StringTransport
from twisted.protocols.policies import WrappingFactory
-from twisted.trial import unittest
from twisted.web import resource, server, static, util
+from twisted.web.client import _makeGetterFactory
from scrapy.core.downloader import webclient as client
-from scrapy.core.downloader.contextfactory import (
- ScrapyClientContextFactory,
-)
+from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
from scrapy.http import Headers, Request
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.utils.test import get_crawler
-from tests.mockserver import (
- BrokenDownloadResource,
- ErrorResource,
+from tests.mockserver.http_resources import (
ForeverTakingResource,
HostHeaderResource,
- NoLengthResource,
PayloadResource,
- ssl_context_factory,
)
+from tests.mockserver.utils import ssl_context_factory
from tests.test_core_downloader import TestContextFactoryBase
@@ -51,8 +43,6 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
f.deferred.addCallback(response_transform or (lambda r: r.body))
return f
- from twisted.web.client import _makeGetterFactory
-
return _makeGetterFactory(
to_bytes(url),
_clientfactory,
@@ -202,17 +192,38 @@ class EncodingResource(resource.Resource):
return body.encode(self.out_encoding)
+class BrokenDownloadResource(resource.Resource):
+ def render(self, request):
+ # only sends 3 bytes even though it claims to send 5
+ request.setHeader(b"content-length", b"5")
+ request.write(b"abc")
+ return b""
+
+
+class ErrorResource(resource.Resource):
+ def render(self, request):
+ request.setResponseCode(401)
+ if request.args.get(b"showlength"):
+ request.setHeader(b"content-length", b"0")
+ return b""
+
+
+class NoLengthResource(resource.Resource):
+ def render(self, request):
+ return b"nolength"
+
+
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestWebClient(unittest.TestCase):
+class TestWebClient:
def _listen(self, site):
from twisted.internet import reactor
return reactor.listenTCP(0, site, interface="127.0.0.1")
- def setUp(self):
- self.tmpname = Path(mkdtemp())
- (self.tmpname / "file").write_bytes(b"0123456789")
- r = static.File(str(self.tmpname))
+ @pytest.fixture
+ def wrapper(self, tmp_path):
+ (tmp_path / "file").write_bytes(b"0123456789")
+ r = static.File(str(tmp_path))
r.putChild(b"redirect", util.Redirect(b"/file"))
r.putChild(b"wait", ForeverTakingResource())
r.putChild(b"error", ErrorResource())
@@ -221,45 +232,47 @@ class TestWebClient(unittest.TestCase):
r.putChild(b"payload", PayloadResource())
r.putChild(b"broken", BrokenDownloadResource())
r.putChild(b"encoding", EncodingResource())
- self.site = server.Site(r, timeout=None)
- self.wrapper = WrappingFactory(self.site)
- self.port = self._listen(self.wrapper)
- self.portno = self.port.getHost().port
+ site = server.Site(r, timeout=None)
+ return WrappingFactory(site)
+
+ @async_yield_fixture
+ async def server_port(self, wrapper):
+ port = self._listen(wrapper)
+
+ yield port.getHost().port
+
+ await port.stopListening()
+
+ @pytest.fixture
+ def server_url(self, server_port):
+ return f"http://127.0.0.1:{server_port}/"
@inlineCallbacks
- def tearDown(self):
- yield self.port.stopListening()
- shutil.rmtree(self.tmpname)
-
- def getURL(self, path):
- return f"http://127.0.0.1:{self.portno}/{path}"
-
- @inlineCallbacks
- def testPayload(self):
+ def testPayload(self, server_url):
s = "0123456789" * 10
- body = yield getPage(self.getURL("payload"), body=s)
+ body = yield getPage(server_url + "payload", body=s)
assert body == to_bytes(s)
@inlineCallbacks
- def testHostHeader(self):
+ def testHostHeader(self, server_port, server_url):
# if we pass Host header explicitly, it should be used, otherwise
# it should extract from url
- body = yield getPage(self.getURL("host"))
- assert body == to_bytes(f"127.0.0.1:{self.portno}")
- body = yield getPage(self.getURL("host"), headers={"Host": "www.example.com"})
+ body = yield getPage(server_url + "host")
+ assert body == to_bytes(f"127.0.0.1:{server_port}")
+ body = yield getPage(server_url + "host", headers={"Host": "www.example.com"})
assert body == to_bytes("www.example.com")
@inlineCallbacks
- def test_getPage(self):
+ def test_getPage(self, server_url):
"""
L{client.getPage} returns a L{Deferred} which is called back with
the body of the response if the default method B{GET} is used.
"""
- body = yield getPage(self.getURL("file"))
+ body = yield getPage(server_url + "file")
assert body == b"0123456789"
@inlineCallbacks
- def test_getPageHead(self):
+ def test_getPageHead(self, server_url):
"""
L{client.getPage} returns a L{Deferred} which is called back with
the empty string if the method is C{HEAD} and there is a successful
@@ -267,7 +280,7 @@ class TestWebClient(unittest.TestCase):
"""
def _getPage(method):
- return getPage(self.getURL("file"), method=method)
+ return getPage(server_url + "file", method=method)
body = yield _getPage("head")
assert body == b""
@@ -275,42 +288,42 @@ class TestWebClient(unittest.TestCase):
assert body == b""
@inlineCallbacks
- def test_timeoutNotTriggering(self):
+ def test_timeoutNotTriggering(self, server_port, server_url):
"""
When a non-zero timeout is passed to L{getPage} and the page is
retrieved before the timeout period elapses, the L{Deferred} is
called back with the contents of the page.
"""
- body = yield getPage(self.getURL("host"), timeout=100)
- assert body == to_bytes(f"127.0.0.1:{self.portno}")
+ body = yield getPage(server_url + "host", timeout=100)
+ assert body == to_bytes(f"127.0.0.1:{server_port}")
@inlineCallbacks
- def test_timeoutTriggering(self):
+ def test_timeoutTriggering(self, wrapper, server_url):
"""
When a non-zero timeout is passed to L{getPage} and that many
seconds elapse before the server responds to the request. the
L{Deferred} is errbacked with a L{error.TimeoutError}.
"""
with pytest.raises(defer.TimeoutError):
- yield getPage(self.getURL("wait"), timeout=0.000001)
+ yield getPage(server_url + "wait", timeout=0.000001)
# Clean up the server which is hanging around not doing
# anything.
- connected = list(self.wrapper.protocols.keys())
+ connected = list(wrapper.protocols.keys())
# There might be nothing here if the server managed to already see
# that the connection was lost.
if connected:
connected[0].transport.loseConnection()
@inlineCallbacks
- def testNotFound(self):
- body = yield getPage(self.getURL("notsuchfile"))
+ def testNotFound(self, server_url):
+ body = yield getPage(server_url + "notsuchfile")
assert b"404 - No Such Resource" in body
@inlineCallbacks
- def testFactoryInfo(self):
+ def testFactoryInfo(self, server_url):
from twisted.internet import reactor
- url = self.getURL("file")
+ url = server_url + "file"
parsed = urlparse(url)
factory = client.ScrapyHTTPClientFactory(Request(url))
reactor.connectTCP(parsed.hostname, parsed.port, factory)
@@ -321,8 +334,8 @@ class TestWebClient(unittest.TestCase):
assert factory.response_headers[b"content-length"] == b"10"
@inlineCallbacks
- def testRedirect(self):
- body = yield getPage(self.getURL("redirect"))
+ def testRedirect(self, server_url):
+ body = yield getPage(server_url + "redirect")
assert (
body
== b'\n\n \n