RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Preparations for switching to direct pytest. (#6678)

This commit is contained in:
Andrey Rakhmatullin 2025-02-16 23:28:58 +04:00 committed by GitHub
parent 7d5b189c11
commit a898331d14
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
41 changed files with 285 additions and 286 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
conftest.py
View File

@ -41,7 +41,7 @@ if not H2_ENABLED:
)
@pytest.fixture()
@pytest.fixture
def chdir(tmpdir):
"""Change to pytest-provided temporary directory"""
tmpdir.chdir()

11
pyproject.toml
View File

@ -200,8 +200,7 @@ disable = [
[tool.pytest.ini_options]
xfail_strict = true
usefixtures = "chdir"
python_files = ["test_*.py", "__init__.py"]
python_classes = []
python_files = ["test_*.py", "test_*/__init__.py"]
addopts = [
"--assert=plain",
"--ignore=docs/_ext",
@ -254,6 +253,8 @@ extend-select = [
"PIE",
# pylint
"PL",
# flake8-pytest-style
"PT",
# flake8-use-pathlib
"PTH",
# flake8-pyi
@ -373,6 +374,12 @@ ignore = [
"B904",
# Use capitalized environment variable
"SIM112",
# Temporarily silenced PT rules
# Use a regular `assert` instead of unittest-style `assertEqual`
"PT009",
# Use `pytest.raises` instead of unittest-style `assertRaises`
"PT027",
]
[tool.ruff.lint.per-file-ignores]

9
scrapy/utils/test.py
View File

@ -15,9 +15,10 @@ from unittest import TestCase, mock
from twisted.trial.unittest import SkipTest
from scrapy import Spider
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.spider import DefaultSpider
if TYPE_CHECKING:
from collections.abc import Awaitable
@ -25,6 +26,7 @@ if TYPE_CHECKING:
from twisted.internet.defer import Deferred
from twisted.web.client import Response as TxResponse
from scrapy import Spider
from scrapy.crawler import Crawler
@ -82,8 +84,7 @@ def get_ftp_content_and_delete(
return b"".join(ftp_data)
class TestSpider(Spider):
name = "test"
TestSpider = create_deprecated_class("TestSpider", DefaultSpider)
def get_crawler(
@ -101,7 +102,7 @@ def get_crawler(
settings: dict[str, Any] = {}
settings.update(settings_dict or {})
runner = CrawlerRunner(settings)
crawler = runner.create_crawler(spidercls or TestSpider)
crawler = runner.create_crawler(spidercls or DefaultSpider)
crawler._apply_settings()
return crawler

4
tests/test_commands.py
View File

@ -19,7 +19,7 @@ from threading import Timer
from typing import TYPE_CHECKING
from unittest import mock, skipIf
from pytest import mark
import pytest
from twisted.trial import unittest
import scrapy
@ -822,7 +822,7 @@ class MySpider(scrapy.Spider):
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
)
@mark.requires_uvloop
@pytest.mark.requires_uvloop
def test_custom_asyncio_loop_enabled_true(self):
log = self.get_log(
self.debug_log_spider,

50
tests/test_contracts.py
View File

@ -21,7 +21,7 @@ from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
class TestItem(Item):
class DemoItem(Item):
name = Field()
url = Field()
@ -58,7 +58,7 @@ class CustomFormContract(Contract):
return args
class TestSpider(Spider):
class DemoSpider(Spider):
name = "demo_spider"
def returns_request(self, response):
@ -80,7 +80,7 @@ class TestSpider(Spider):
@url http://scrapy.org
@returns items 1 1
"""
return TestItem(url=response.url)
return DemoItem(url=response.url)
def returns_request_cb_kwargs(self, response, url):
"""method which returns request
@ -96,7 +96,7 @@ class TestSpider(Spider):
@cb_kwargs {"name": "Scrapy"}
@returns items 1 1
"""
return TestItem(name=name, url=response.url)
return DemoItem(name=name, url=response.url)
def returns_item_cb_kwargs_error_unexpected_keyword(self, response):
"""method which returns item
@ -104,14 +104,14 @@ class TestSpider(Spider):
@cb_kwargs {"arg": "value"}
@returns items 1 1
"""
return TestItem(url=response.url)
return DemoItem(url=response.url)
def returns_item_cb_kwargs_error_missing_argument(self, response, arg):
"""method which returns item
@url http://scrapy.org
@returns items 1 1
"""
return TestItem(url=response.url)
return DemoItem(url=response.url)
def returns_dict_item(self, response):
"""method which returns item
@ -125,7 +125,7 @@ class TestSpider(Spider):
@url http://scrapy.org
@returns items 0 0
"""
return TestItem(url=response.url)
return DemoItem(url=response.url)
def returns_dict_fail(self, response):
"""method which returns item
@ -140,7 +140,7 @@ class TestSpider(Spider):
@returns items 1 1
@scrapes name url
"""
return TestItem(name="test", url=response.url)
return DemoItem(name="test", url=response.url)
def scrapes_dict_item_ok(self, response):
"""returns item with name and url
@ -156,7 +156,7 @@ class TestSpider(Spider):
@returns items 1 1
@scrapes name url
"""
return TestItem(url=response.url)
return DemoItem(url=response.url)
def scrapes_dict_item_fail(self, response):
"""returns item with no name
@ -212,7 +212,7 @@ class TestSpider(Spider):
@meta {"key": "example"}
@returns items 1 1
"""
return TestItem(name="example", url=response.url)
return DemoItem(name="example", url=response.url)
def returns_error_missing_meta(self, response):
"""method which depends of metadata be defined
@ -242,7 +242,7 @@ class CustomContractFailSpider(Spider):
"""
class InheritsTestSpider(TestSpider):
class InheritsDemoSpider(DemoSpider):
name = "inherits_demo_spider"
@ -274,7 +274,7 @@ class ContractsManagerTest(unittest.TestCase):
self.assertTrue(self.results.errors)
def test_contracts(self):
spider = TestSpider()
spider = DemoSpider()
# extract contracts correctly
contracts = self.conman.extract_contracts(spider.returns_request)
@ -293,7 +293,7 @@ class ContractsManagerTest(unittest.TestCase):
self.assertEqual(request, None)
def test_cb_kwargs(self):
spider = TestSpider()
spider = DemoSpider()
response = ResponseMock()
# extract contracts correctly
@ -356,7 +356,7 @@ class ContractsManagerTest(unittest.TestCase):
self.should_error()
def test_meta(self):
spider = TestSpider()
spider = DemoSpider()
# extract contracts correctly
contracts = self.conman.extract_contracts(spider.returns_request_meta)
@ -402,7 +402,7 @@ class ContractsManagerTest(unittest.TestCase):
self.should_error()
def test_returns(self):
spider = TestSpider()
spider = DemoSpider()
response = ResponseMock()
# returns_item
@ -431,7 +431,7 @@ class ContractsManagerTest(unittest.TestCase):
self.should_fail()
def test_returns_async(self):
spider = TestSpider()
spider = DemoSpider()
response = ResponseMock()
request = self.conman.from_method(spider.returns_request_async, self.results)
@ -439,7 +439,7 @@ class ContractsManagerTest(unittest.TestCase):
self.should_error()
def test_scrapes(self):
spider = TestSpider()
spider = DemoSpider()
response = ResponseMock()
# scrapes_item_ok
@ -472,7 +472,7 @@ class ContractsManagerTest(unittest.TestCase):
assert message in self.results.failures[-1][-1]
def test_regex(self):
spider = TestSpider()
spider = DemoSpider()
response = ResponseMock()
# invalid regex
@ -494,7 +494,7 @@ class ContractsManagerTest(unittest.TestCase):
self.should_error()
def test_errback(self):
spider = TestSpider()
spider = DemoSpider()
response = ResponseMock()
try:
@ -522,11 +522,11 @@ class ContractsManagerTest(unittest.TestCase):
def parse_first(self, response):
self.visited += 1
return TestItem()
return DemoItem()
def parse_second(self, response):
self.visited += 1
return TestItem()
return DemoItem()
with MockServer() as mockserver:
contract_doc = f"@url {mockserver.url('/status?n=200')}"
@ -540,13 +540,13 @@ class ContractsManagerTest(unittest.TestCase):
self.assertEqual(crawler.spider.visited, 2)
def test_form_contract(self):
spider = TestSpider()
spider = DemoSpider()
request = self.conman.from_method(spider.custom_form, self.results)
self.assertEqual(request.method, "POST")
self.assertIsInstance(request, FormRequest)
def test_inherited_contracts(self):
spider = InheritsTestSpider()
spider = InheritsDemoSpider()
requests = self.conman.from_spider(spider, self.results)
self.assertTrue(requests)
@ -571,7 +571,7 @@ class CustomContractPrePostProcess(unittest.TestCase):
self.results = TextTestResult(stream=None, descriptions=False, verbosity=0)
def test_pre_hook_keyboard_interrupt(self):
spider = TestSpider()
spider = DemoSpider()
response = ResponseMock()
contract = CustomFailContractPreProcess(spider.returns_request)
conman = ContractsManager([contract])
@ -590,7 +590,7 @@ class CustomContractPrePostProcess(unittest.TestCase):
self.assertFalse(self.results.errors)
def test_post_hook_keyboard_interrupt(self):
spider = TestSpider()
spider = DemoSpider()
response = ResponseMock()
contract = CustomFailContractPostProcess(spider.returns_request)
conman = ContractsManager([contract])

26
tests/test_crawl.py
View File

@ -6,7 +6,7 @@ from ipaddress import IPv4Address
from socket import gethostbyname
from urllib.parse import urlparse
from pytest import mark
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.ssl import Certificate
@ -536,7 +536,7 @@ class CrawlSpiderTestCase(TestCase):
)
self.assertIn("Got response 200", str(log))
@mark.only_asyncio()
@pytest.mark.only_asyncio
@defer.inlineCallbacks
def test_async_def_asyncio_parse(self):
crawler = get_crawler(
@ -551,7 +551,7 @@ class CrawlSpiderTestCase(TestCase):
)
self.assertIn("Got response 200", str(log))
@mark.only_asyncio()
@pytest.mark.only_asyncio
@defer.inlineCallbacks
def test_async_def_asyncio_parse_items_list(self):
log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider)
@ -559,7 +559,7 @@ class CrawlSpiderTestCase(TestCase):
self.assertIn({"id": 1}, items)
self.assertIn({"id": 2}, items)
@mark.only_asyncio()
@pytest.mark.only_asyncio
@defer.inlineCallbacks
def test_async_def_asyncio_parse_items_single_element(self):
items = []
@ -576,7 +576,7 @@ class CrawlSpiderTestCase(TestCase):
self.assertIn("Got response 200", str(log))
self.assertIn({"foo": 42}, items)
@mark.only_asyncio()
@pytest.mark.only_asyncio
@defer.inlineCallbacks
def test_async_def_asyncgen_parse(self):
log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider)
@ -584,7 +584,7 @@ class CrawlSpiderTestCase(TestCase):
itemcount = stats.get_value("item_scraped_count")
self.assertEqual(itemcount, 1)
@mark.only_asyncio()
@pytest.mark.only_asyncio
@defer.inlineCallbacks
def test_async_def_asyncgen_parse_loop(self):
log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider)
@ -594,7 +594,7 @@ class CrawlSpiderTestCase(TestCase):
for i in range(10):
self.assertIn({"foo": i}, items)
@mark.only_asyncio()
@pytest.mark.only_asyncio
@defer.inlineCallbacks
def test_async_def_asyncgen_parse_exc(self):
log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider)
@ -606,7 +606,7 @@ class CrawlSpiderTestCase(TestCase):
for i in range(7):
self.assertIn({"foo": i}, items)
@mark.only_asyncio()
@pytest.mark.only_asyncio
@defer.inlineCallbacks
def test_async_def_asyncgen_parse_complex(self):
_, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider)
@ -618,20 +618,20 @@ class CrawlSpiderTestCase(TestCase):
for i in [10, 30, 122]:
self.assertIn({"index2": i}, items)
@mark.only_asyncio()
@pytest.mark.only_asyncio
@defer.inlineCallbacks
def test_async_def_asyncio_parse_reqs_list(self):
log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider)
for req_id in range(3):
self.assertIn(f"Got response 200, req_id {req_id}", str(log))
@mark.only_not_asyncio()
@pytest.mark.only_not_asyncio
@defer.inlineCallbacks
def test_async_def_deferred_direct(self):
_, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider)
self.assertEqual(items, [{"code": 200}])
@mark.only_asyncio()
@pytest.mark.only_asyncio
@defer.inlineCallbacks
def test_async_def_deferred_wrapped(self):
log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider)
@ -659,7 +659,9 @@ class CrawlSpiderTestCase(TestCase):
self.assertEqual(cert.getSubject().commonName, b"localhost")
self.assertEqual(cert.getIssuer().commonName, b"localhost")
@mark.xfail(reason="Responses with no body return early and contain no certificate")
@pytest.mark.xfail(
reason="Responses with no body return early and contain no certificate"
)
@defer.inlineCallbacks
def test_response_ssl_certificate_empty_response(self):
crawler = get_crawler(SingleRequestSpider)

36
tests/test_crawler.py
View File

@ -8,9 +8,9 @@ import warnings
from pathlib import Path
from typing import Any
import pytest
from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn
from pytest import mark, raises
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.trial import unittest
from w3lib import __version__ as w3lib_version
@ -77,14 +77,14 @@ class CrawlerTestCase(BaseCrawlerTest):
self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
def test_crawler_rejects_spider_objects(self):
with raises(ValueError):
with pytest.raises(ValueError, match="spidercls argument must be a class"):
Crawler(DefaultSpider())
@inlineCallbacks
def test_crawler_crawl_twice_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
yield crawler.crawl()
with raises(RuntimeError, match="more than once on the same instance"):
with pytest.raises(RuntimeError, match="more than once on the same instance"):
yield crawler.crawl()
def test_get_addon(self):
@ -203,7 +203,7 @@ class CrawlerTestCase(BaseCrawlerTest):
raise
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with raises(RuntimeError):
with pytest.raises(RuntimeError):
yield crawler.crawl()
@inlineCallbacks
@ -282,7 +282,7 @@ class CrawlerTestCase(BaseCrawlerTest):
raise
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with raises(RuntimeError):
with pytest.raises(RuntimeError):
yield crawler.crawl()
@inlineCallbacks
@ -361,7 +361,7 @@ class CrawlerTestCase(BaseCrawlerTest):
raise
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with raises(RuntimeError):
with pytest.raises(RuntimeError):
yield crawler.crawl()
@inlineCallbacks
@ -440,7 +440,7 @@ class CrawlerTestCase(BaseCrawlerTest):
raise
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with raises(RuntimeError):
with pytest.raises(RuntimeError):
yield crawler.crawl()
@ -575,7 +575,7 @@ class NoRequestsSpider(scrapy.Spider):
return []
@mark.usefixtures("reactor_pytest")
@pytest.mark.usefixtures("reactor_pytest")
class CrawlerRunnerHasSpider(unittest.TestCase):
def _runner(self):
return CrawlerRunner()
@ -744,7 +744,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
)
@mark.skipif(
@pytest.mark.skipif(
parse_version(w3lib_version) >= parse_version("2.0.0"),
reason="w3lib 2.0.0 and later do not allow invalid domains.",
)
@ -781,7 +781,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
"Using reactor: twisted.internet.selectreactor.SelectReactor", log
)
@mark.skipif(
@pytest.mark.skipif(
platform.system() == "Windows", reason="PollReactor is not supported on Windows"
)
def test_twisted_reactor_poll(self):
@ -820,7 +820,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
log,
)
@mark.requires_uvloop
@pytest.mark.requires_uvloop
def test_custom_loop_asyncio(self):
log = self.run_script("asyncio_custom_loop.py")
self.assertIn("Spider closed (finished)", log)
@ -829,7 +829,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
)
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
@mark.requires_uvloop
@pytest.mark.requires_uvloop
def test_custom_loop_asyncio_deferred_signal(self):
log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
self.assertIn("Spider closed (finished)", log)
@ -839,7 +839,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
self.assertIn("async pipeline opened!", log)
@mark.requires_uvloop
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_same_loop(self):
log = self.run_script("asyncio_enabled_reactor_same_loop.py")
self.assertIn("Spider closed (finished)", log)
@ -848,7 +848,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
)
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
@mark.requires_uvloop
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_different_loop(self):
log = self.run_script("asyncio_enabled_reactor_different_loop.py")
self.assertNotIn("Spider closed (finished)", log)
@ -924,13 +924,13 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
self.assertIn("DEBUG: Using asyncio event loop", log)
@mark.parametrize(
["settings", "items"],
(
@pytest.mark.parametrize(
("settings", "items"),
[
({}, default_settings.LOG_VERSIONS),
({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]),
({"LOG_VERSIONS": []}, None),
),
],
)
def test_log_scrapy_info(settings, items, caplog):
with caplog.at_level("INFO"):

6
tests/test_downloadermiddleware.py
View File

@ -1,7 +1,7 @@
import asyncio
from unittest import mock
from pytest import mark
import pytest
from twisted.internet import defer
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
@ -220,7 +220,7 @@ class MiddlewareUsingDeferreds(ManagerTestCase):
self.assertFalse(download_func.called)
@mark.usefixtures("reactor_pytest")
@pytest.mark.usefixtures("reactor_pytest")
class MiddlewareUsingCoro(ManagerTestCase):
"""Middlewares using asyncio coroutines should work"""
@ -243,7 +243,7 @@ class MiddlewareUsingCoro(ManagerTestCase):
self.assertIs(results[0], resp)
self.assertFalse(download_func.called)
@mark.only_asyncio()
@pytest.mark.only_asyncio
def test_asyncdef_asyncio(self):
resp = Response("http://example.com/index.html")

2
tests/test_downloadermiddleware_ajaxcrawlable.py
View File

@ -7,8 +7,6 @@ from scrapy.http import HtmlResponse, Request, Response
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
__doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"]
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class AjaxCrawlMiddlewareTest(unittest.TestCase):

12
tests/test_downloadermiddleware_httpauth.py
View File

@ -7,18 +7,18 @@ from scrapy.http import Request
from scrapy.spiders import Spider
class TestSpiderLegacy(Spider):
class LegacySpider(Spider):
http_user = "foo"
http_pass = "bar"
class TestSpider(Spider):
class DomainSpider(Spider):
http_user = "foo"
http_pass = "bar"
http_auth_domain = "example.com"
class TestSpiderAny(Spider):
class AnyDomainSpider(Spider):
http_user = "foo"
http_pass = "bar"
http_auth_domain = None
@ -26,7 +26,7 @@ class TestSpiderAny(Spider):
class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
def setUp(self):
self.spider = TestSpiderLegacy("foo")
self.spider = LegacySpider("foo")
def test_auth(self):
with self.assertRaises(AttributeError):
@ -37,7 +37,7 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
class HttpAuthMiddlewareTest(unittest.TestCase):
def setUp(self):
self.mw = HttpAuthMiddleware()
self.spider = TestSpider("foo")
self.spider = DomainSpider("foo")
self.mw.spider_opened(self.spider)
def tearDown(self):
@ -67,7 +67,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase):
class HttpAuthAnyMiddlewareTest(unittest.TestCase):
def setUp(self):
self.mw = HttpAuthMiddleware()
self.spider = TestSpiderAny("foo")
self.spider = AnyDomainSpider("foo")
self.mw.spider_opened(self.spider)
def tearDown(self):

6
tests/test_downloadermiddleware_httpcache.py
View File

@ -353,7 +353,8 @@ class RFC2616PolicyTest(DefaultStorageTest):
resc = mw.storage.retrieve_response(self.spider, req0)
if shouldcache:
self.assertEqualResponse(resc, res1)
assert "cached" in res2.flags and res2.status != 304
assert "cached" in res2.flags
assert res2.status != 304
else:
self.assertFalse(resc)
assert "cached" not in res2.flags
@ -376,7 +377,8 @@ class RFC2616PolicyTest(DefaultStorageTest):
resc = mw.storage.retrieve_response(self.spider, req0)
if shouldcache:
self.assertEqualResponse(resc, res1)
assert "cached" in res2.flags and res2.status != 304
assert "cached" in res2.flags
assert res2.status != 304
else:
self.assertFalse(resc)
assert "cached" not in res2.flags

3
tests/test_downloadermiddleware_httpproxy.py
View File

@ -131,7 +131,8 @@ class TestHttpProxyMiddleware(TestCase):
mw = HttpProxyMiddleware()
req = Request("http://noproxy.com", meta={"proxy": None})
assert mw.process_request(req, spider) is None
assert "proxy" in req.meta and req.meta["proxy"] is None
assert "proxy" in req.meta
assert req.meta["proxy"] is None
def test_no_proxy(self):
os.environ["http_proxy"] = "https://proxy.for.http:3128"

28
tests/test_downloadermiddleware_offsite.py
View File

@ -12,7 +12,7 @@ UNSET = object()
@pytest.mark.parametrize(
("allowed_domain", "url", "allowed"),
(
[
("example.com", "http://example.com/1", True),
("example.com", "http://example.org/1", False),
("example.com", "http://sub.example.com/1", True),
@ -24,7 +24,7 @@ UNSET = object()
("example.com", "http://example.com.example", False),
("a.example", "http://nota.example", False),
("b.a.example", "http://notb.a.example", False),
),
],
)
def test_process_request_domain_filtering(allowed_domain, url, allowed):
crawler = get_crawler(Spider)
@ -41,12 +41,12 @@ def test_process_request_domain_filtering(allowed_domain, url, allowed):
@pytest.mark.parametrize(
("value", "filtered"),
(
[
(UNSET, True),
(None, True),
(False, True),
(True, False),
),
],
)
def test_process_request_dont_filter(value, filtered):
crawler = get_crawler(Spider)
@ -66,7 +66,7 @@ def test_process_request_dont_filter(value, filtered):
@pytest.mark.parametrize(
("allow_offsite", "dont_filter", "filtered"),
(
[
(True, UNSET, False),
(True, None, False),
(True, False, False),
@ -75,7 +75,7 @@ def test_process_request_dont_filter(value, filtered):
(False, None, True),
(False, False, True),
(False, True, False),
),
],
)
def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
crawler = get_crawler(Spider)
@ -97,11 +97,11 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
@pytest.mark.parametrize(
"value",
(
[
UNSET,
None,
[],
),
],
)
def test_process_request_no_allowed_domains(value):
crawler = get_crawler(Spider)
@ -133,7 +133,7 @@ def test_process_request_invalid_domains():
@pytest.mark.parametrize(
("allowed_domain", "url", "allowed"),
(
[
("example.com", "http://example.com/1", True),
("example.com", "http://example.org/1", False),
("example.com", "http://sub.example.com/1", True),
@ -145,7 +145,7 @@ def test_process_request_invalid_domains():
("example.com", "http://example.com.example", False),
("a.example", "http://nota.example", False),
("b.a.example", "http://notb.a.example", False),
),
],
)
def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
crawler = get_crawler(Spider)
@ -162,12 +162,12 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
@pytest.mark.parametrize(
("value", "filtered"),
(
[
(UNSET, True),
(None, True),
(False, True),
(True, False),
),
],
)
def test_request_scheduled_dont_filter(value, filtered):
crawler = get_crawler(Spider)
@ -187,11 +187,11 @@ def test_request_scheduled_dont_filter(value, filtered):
@pytest.mark.parametrize(
"value",
(
[
UNSET,
None,
[],
),
],
)
def test_request_scheduled_no_allowed_domains(value):
crawler = get_crawler(Spider)

4
tests/test_downloadermiddleware_redirect.py
View File

@ -1278,7 +1278,7 @@ class MetaRefreshMiddlewareTest(Base.Test):
@pytest.mark.parametrize(
SCHEME_PARAMS,
(
[
*REDIRECT_SCHEME_CASES,
# data/file/ftp/s3/foo → * does not redirect
*(
@ -1300,7 +1300,7 @@ class MetaRefreshMiddlewareTest(Base.Test):
for scheme in NON_HTTP_SCHEMES
for location in ("//example.com/b", "/b")
),
),
],
)
def test_meta_refresh_schemes(url, location, target):
crawler = get_crawler(Spider)

30
tests/test_engine.py
View File

@ -42,7 +42,7 @@ from scrapy.utils.test import get_crawler
from tests import get_testdata, tests_datadir
class TestItem(Item):
class MyItem(Item):
name = Field()
url = Field()
price = Field()
@ -62,7 +62,7 @@ class DataClassItem:
price: int = 0
class TestSpider(Spider):
class MySpider(Spider):
name = "scrapytest.org"
allowed_domains = ["scrapytest.org", "localhost"]
@ -70,7 +70,7 @@ class TestSpider(Spider):
name_re = re.compile(r"<h1>(.*?)</h1>", re.MULTILINE)
price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE)
item_cls: type = TestItem
item_cls: type = MyItem
def parse(self, response):
xlink = LinkExtractor()
@ -91,24 +91,24 @@ class TestSpider(Spider):
return adapter.item
class TestDupeFilterSpider(TestSpider):
class DupeFilterSpider(MySpider):
def start_requests(self):
return (Request(url) for url in self.start_urls) # no dont_filter=True
class DictItemsSpider(TestSpider):
class DictItemsSpider(MySpider):
item_cls = dict
class AttrsItemsSpider(TestSpider):
class AttrsItemsSpider(MySpider):
item_cls = AttrsItem
class DataClassItemsSpider(TestSpider):
class DataClassItemsSpider(MySpider):
item_cls = DataClassItem
class ItemZeroDivisionErrorSpider(TestSpider):
class ItemZeroDivisionErrorSpider(MySpider):
custom_settings = {
"ITEM_PIPELINES": {
"tests.pipelines.ProcessWithZeroDivisionErrorPipeline": 300,
@ -116,7 +116,7 @@ class ItemZeroDivisionErrorSpider(TestSpider):
}
class ChangeCloseReasonSpider(TestSpider):
class ChangeCloseReasonSpider(MySpider):
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = cls(*args, **kwargs)
@ -388,7 +388,7 @@ class EngineTest(EngineTestBase):
@defer.inlineCallbacks
def test_crawler(self):
for spider in (
TestSpider,
MySpider,
DictItemsSpider,
AttrsItemsSpider,
DataClassItemsSpider,
@ -404,7 +404,7 @@ class EngineTest(EngineTestBase):
@defer.inlineCallbacks
def test_crawler_dupefilter(self):
run = CrawlerRun(TestDupeFilterSpider)
run = CrawlerRun(DupeFilterSpider)
yield run.run()
self._assert_scheduled_requests(run, count=8)
self._assert_dropped_requests(run)
@ -426,13 +426,13 @@ class EngineTest(EngineTestBase):
@defer.inlineCallbacks
def test_close_downloader(self):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
e = ExecutionEngine(get_crawler(MySpider), lambda _: None)
yield e.close()
@defer.inlineCallbacks
def test_start_already_running_exception(self):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
e = ExecutionEngine(get_crawler(MySpider), lambda _: None)
yield e.open_spider(MySpider(), [])
e.start()
try:
yield self.assertFailure(e.start(), RuntimeError).addBoth(
@ -486,7 +486,7 @@ def test_request_scheduled_signal(caplog):
if "drop" in request.url:
raise IgnoreRequest
spider = TestSpider()
spider = MySpider()
crawler = get_crawler(spider.__class__)
engine = ExecutionEngine(crawler, lambda _: None)
engine.downloader._slot_gc_loop.stop()

4
tests/test_engine_stop_download_bytes.py
View File

@ -8,7 +8,7 @@ from tests.test_engine import (
DataClassItemsSpider,
DictItemsSpider,
EngineTestBase,
TestSpider,
MySpider,
)
@ -22,7 +22,7 @@ class BytesReceivedEngineTest(EngineTestBase):
@defer.inlineCallbacks
def test_crawler(self):
for spider in (
TestSpider,
MySpider,
DictItemsSpider,
AttrsItemsSpider,
DataClassItemsSpider,

4
tests/test_engine_stop_download_headers.py
View File

@ -8,7 +8,7 @@ from tests.test_engine import (
DataClassItemsSpider,
DictItemsSpider,
EngineTestBase,
TestSpider,
MySpider,
)
@ -22,7 +22,7 @@ class HeadersReceivedEngineTest(EngineTestBase):
@defer.inlineCallbacks
def test_crawler(self):
for spider in (
TestSpider,
MySpider,
DictItemsSpider,
AttrsItemsSpider,
DataClassItemsSpider,

40
tests/test_exporters.py
View File

@ -31,7 +31,7 @@ def custom_serializer(value):
return str(int(value) + 2)
class TestItem(Item):
class MyItem(Item):
name = Field()
age = Field()
@ -42,7 +42,7 @@ class CustomFieldItem(Item):
@dataclasses.dataclass
class TestDataClass:
class MyDataClass:
name: str
age: int
@ -54,7 +54,7 @@ class CustomFieldDataclass:
class BaseItemExporterTest(unittest.TestCase):
item_class: type = TestItem
item_class: type = MyItem
custom_field_item_class: type = CustomFieldItem
def setUp(self):
@ -138,7 +138,7 @@ class BaseItemExporterTest(unittest.TestCase):
class BaseItemExporterDataclassTest(BaseItemExporterTest):
item_class = TestDataClass
item_class = MyDataClass
custom_field_item_class = CustomFieldDataclass
@ -207,7 +207,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
class PythonItemExporterDataclassTest(PythonItemExporterTest):
item_class = TestDataClass
item_class = MyDataClass
custom_field_item_class = CustomFieldDataclass
@ -222,7 +222,7 @@ class PprintItemExporterTest(BaseItemExporterTest):
class PprintItemExporterDataclassTest(PprintItemExporterTest):
item_class = TestDataClass
item_class = MyDataClass
custom_field_item_class = CustomFieldDataclass
@ -259,7 +259,7 @@ class PickleItemExporterTest(BaseItemExporterTest):
class PickleItemExporterDataclassTest(PickleItemExporterTest):
item_class = TestDataClass
item_class = MyDataClass
custom_field_item_class = CustomFieldDataclass
@ -286,7 +286,7 @@ class MarshalItemExporterTest(BaseItemExporterTest):
class MarshalItemExporterDataclassTest(MarshalItemExporterTest):
item_class = TestDataClass
item_class = MyDataClass
custom_field_item_class = CustomFieldDataclass
@ -406,7 +406,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
class CsvItemExporterDataclassTest(CsvItemExporterTest):
item_class = TestDataClass
item_class = MyDataClass
custom_field_item_class = CustomFieldDataclass
@ -517,7 +517,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
class XmlItemExporterDataclassTest(XmlItemExporterTest):
item_class = TestDataClass
item_class = MyDataClass
custom_field_item_class = CustomFieldDataclass
@ -563,7 +563,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest):
item_class = TestDataClass
item_class = MyDataClass
custom_field_item_class = CustomFieldDataclass
@ -595,11 +595,11 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
def test_two_items_with_failure_between(self):
i1 = TestItem(name="Joseph\xa3", age="22")
i2 = TestItem(
i1 = MyItem(name="Joseph\xa3", age="22")
i2 = MyItem(
name="Maria", age=1j
) # Invalid datetimes didn't consistently fail between Python versions
i3 = TestItem(name="Jesus", age="44")
i3 = MyItem(name="Jesus", age="44")
self.ie.start_exporting()
self.ie.export_item(i1)
self.assertRaises(TypeError, self.ie.export_item, i2)
@ -652,9 +652,9 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest):
return JsonItemExporter(self.output, **kwargs)
def test_two_items_with_failure_between(self):
i1 = TestItem(name="Joseph", age="22")
i2 = TestItem(name="\u263a", age="11")
i3 = TestItem(name="Jesus", age="44")
i1 = MyItem(name="Joseph", age="22")
i2 = MyItem(name="\u263a", age="11")
i3 = MyItem(name="Jesus", age="44")
self.ie.start_exporting()
self.ie.export_item(i1)
self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2)
@ -665,12 +665,12 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest):
class JsonItemExporterDataclassTest(JsonItemExporterTest):
item_class = TestDataClass
item_class = MyDataClass
custom_field_item_class = CustomFieldDataclass
class CustomExporterItemTest(unittest.TestCase):
item_class: type = TestItem
item_class: type = MyItem
def setUp(self):
if self.item_class is None:
@ -700,4 +700,4 @@ class CustomExporterItemTest(unittest.TestCase):
class CustomExporterDataclassTest(CustomExporterItemTest):
item_class = TestDataClass
item_class = MyDataClass

4
tests/test_extension_periodic_log.py
View File

@ -51,7 +51,7 @@ stats_dump_2 = {
}
class TestExtPeriodicLog(PeriodicLog):
class CustomPeriodicLog(PeriodicLog):
def set_a(self):
self.stats._stats = stats_dump_1
@ -62,7 +62,7 @@ class TestExtPeriodicLog(PeriodicLog):
def extension(settings=None):
crawler = Crawler(MetaSpider, settings=settings)
crawler._apply_settings()
return TestExtPeriodicLog.from_crawler(crawler)
return CustomPeriodicLog.from_crawler(crawler)
class TestPeriodicLog(unittest.TestCase):

55
tests/test_extension_throttle.py
View File

@ -13,15 +13,12 @@ from scrapy.settings.default_settings import (
DOWNLOAD_DELAY,
)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler as _get_crawler
UNSET = object()
class TestSpider(Spider):
name = "test"
def get_crawler(settings=None, spidercls=None):
settings = settings or {}
settings["AUTOTHROTTLE_ENABLED"] = True
@ -30,11 +27,11 @@ def get_crawler(settings=None, spidercls=None):
@pytest.mark.parametrize(
("value", "expected"),
(
[
(UNSET, False),
(False, False),
(True, True),
),
],
)
def test_enabled(value, expected):
settings = {}
@ -50,10 +47,10 @@ def test_enabled(value, expected):
@pytest.mark.parametrize(
"value",
(
[
0.0,
-1.0,
),
],
)
def test_target_concurrency_invalid(value):
settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": value}
@ -64,13 +61,13 @@ def test_target_concurrency_invalid(value):
@pytest.mark.parametrize(
("spider", "setting", "expected"),
(
[
(UNSET, UNSET, DOWNLOAD_DELAY),
(1.0, UNSET, 1.0),
(UNSET, 1.0, 1.0),
(1.0, 2.0, 1.0),
(3.0, 2.0, 3.0),
),
],
)
def test_mindelay_definition(spider, setting, expected):
settings = {}
@ -91,10 +88,10 @@ def test_mindelay_definition(spider, setting, expected):
@pytest.mark.parametrize(
("value", "expected"),
(
[
(UNSET, AUTOTHROTTLE_MAX_DELAY),
(1.0, 1.0),
),
],
)
def test_maxdelay_definition(value, expected):
settings = {}
@ -102,13 +99,13 @@ def test_maxdelay_definition(value, expected):
settings["AUTOTHROTTLE_MAX_DELAY"] = value
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
at._spider_opened(TestSpider())
at._spider_opened(DefaultSpider())
assert at.maxdelay == expected
@pytest.mark.parametrize(
("min_spider", "min_setting", "start_setting", "expected"),
(
[
(UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY),
(AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY),
(AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0),
@ -134,7 +131,7 @@ def test_maxdelay_definition(value, expected):
AUTOTHROTTLE_START_DELAY + 2.0,
AUTOTHROTTLE_START_DELAY + 2.0,
),
),
],
)
def test_startdelay_definition(min_spider, min_setting, start_setting, expected):
settings = {}
@ -158,7 +155,7 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected)
@pytest.mark.parametrize(
("meta", "slot"),
(
[
({}, None),
({"download_latency": 1.0}, None),
({"download_slot": "foo"}, None),
@ -172,12 +169,12 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected)
},
"foo",
),
),
],
)
def test_skipped(meta, slot):
crawler = get_crawler()
at = build_from_crawler(AutoThrottle, crawler)
spider = TestSpider()
spider = DefaultSpider()
at._spider_opened(spider)
request = Request("https://example.com", meta=meta)
@ -193,7 +190,7 @@ def test_skipped(meta, slot):
@pytest.mark.parametrize(
("download_latency", "target_concurrency", "slot_delay", "expected"),
(
[
(2.0, 2.0, 1.0, 1.0),
(1.0, 2.0, 1.0, 0.75),
(4.0, 2.0, 1.0, 2.0),
@ -201,13 +198,13 @@ def test_skipped(meta, slot):
(2.0, 4.0, 1.0, 0.75),
(2.0, 2.0, 0.5, 1.0),
(2.0, 2.0, 2.0, 1.5),
),
],
)
def test_adjustment(download_latency, target_concurrency, slot_delay, expected):
settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency}
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
spider = TestSpider()
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
@ -227,11 +224,11 @@ def test_adjustment(download_latency, target_concurrency, slot_delay, expected):
@pytest.mark.parametrize(
("mindelay", "maxdelay", "expected"),
(
[
(0.5, 2.0, 1.0),
(0.25, 0.5, 0.5),
(2.0, 4.0, 2.0),
),
],
)
def test_adjustment_limits(mindelay, maxdelay, expected):
download_latency, target_concurrency, slot_delay = (2.0, 2.0, 1.0)
@ -243,7 +240,7 @@ def test_adjustment_limits(mindelay, maxdelay, expected):
}
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
spider = TestSpider()
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
@ -263,11 +260,11 @@ def test_adjustment_limits(mindelay, maxdelay, expected):
@pytest.mark.parametrize(
("download_latency", "target_concurrency", "slot_delay", "expected"),
(
[
(2.0, 2.0, 1.0, 1.0),
(1.0, 2.0, 1.0, 1.0), # Instead of 0.75
(4.0, 2.0, 1.0, 2.0),
),
],
)
def test_adjustment_bad_response(
download_latency, target_concurrency, slot_delay, expected
@ -275,7 +272,7 @@ def test_adjustment_bad_response(
settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency}
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
spider = TestSpider()
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
@ -297,7 +294,7 @@ def test_debug(caplog):
settings = {"AUTOTHROTTLE_DEBUG": True}
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
spider = TestSpider()
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": 1.0, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
@ -327,7 +324,7 @@ def test_debug(caplog):
def test_debug_disabled(caplog):
crawler = get_crawler()
at = build_from_crawler(AutoThrottle, crawler)
spider = TestSpider()
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": 1.0, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)

3
tests/test_http2_client_protocol.py
View File

@ -258,7 +258,8 @@ class Https2ClientProtocolTestCase(TestCase):
:param path: Should have / at the starting compulsorily if not empty
:return: Complete url
"""
assert len(path) > 0 and (path[0] == "/" or path[0] == "&")
assert len(path) > 0
assert path[0] == "/" or path[0] == "&"
return f"{self.scheme}://{self.hostname}:{self.port_number}{path}"
def make_request(self, request: Request) -> Deferred:

12
tests/test_http_response.py
View File

@ -2,8 +2,8 @@ import codecs
import unittest
from unittest import mock
import pytest
from packaging.version import Version as parse_version
from pytest import mark
from w3lib import __version__ as w3lib_version
from w3lib.encoding import resolve_encoding
@ -218,7 +218,7 @@ class BaseResponseTest(unittest.TestCase):
r = self.response_class("http://example.com")
self.assertRaises(ValueError, r.follow, None)
@mark.xfail(
@pytest.mark.xfail(
parse_version(w3lib_version) < parse_version("2.1.1"),
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
@ -226,7 +226,7 @@ class BaseResponseTest(unittest.TestCase):
def test_follow_whitespace_url(self):
self._assert_followed_url("foo ", "http://example.com/foo")
@mark.xfail(
@pytest.mark.xfail(
parse_version(w3lib_version) < parse_version("2.1.1"),
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
@ -473,10 +473,8 @@ class TextResponseTest(BaseResponseTest):
self._assert_response_encoding(r5, "utf-8")
self._assert_response_encoding(r8, "utf-8")
self._assert_response_encoding(r9, "cp1252")
assert (
r4._body_inferred_encoding() is not None
and r4._body_inferred_encoding() != "ascii"
)
assert r4._body_inferred_encoding() is not None
assert r4._body_inferred_encoding() != "ascii"
self._assert_response_values(r1, "utf-8", "\xa3")
self._assert_response_values(r2, "utf-8", "\xa3")
self._assert_response_values(r3, "iso-8859-1", "\xa3")

4
tests/test_linkextractors.py
View File

@ -4,8 +4,8 @@ import pickle
import re
import unittest
import pytest
from packaging.version import Version
from pytest import mark
from w3lib import __version__ as w3lib_version
from scrapy.http import HtmlResponse, XmlResponse
@ -930,7 +930,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
],
)
@mark.skipif(
@pytest.mark.skipif(
Version(w3lib_version) < Version("2.0.0"),
reason=(
"Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain "

54
tests/test_loader.py
View File

@ -18,12 +18,12 @@ class NameItem(Item):
name = Field()
class TestItem(NameItem):
class SummaryItem(NameItem):
url = Field()
summary = Field()
class TestNestedItem(Item):
class NestedItem(Item):
name = Field()
name_div = Field()
name_value = Field()
@ -38,20 +38,20 @@ class AttrsNameItem:
@dataclasses.dataclass
class TestDataClass:
class NameDataClass:
name: list = dataclasses.field(default_factory=list)
# test item loaders
class NameItemLoader(ItemLoader):
default_item_class = TestItem
default_item_class = SummaryItem
class NestedItemLoader(ItemLoader):
default_item_class = TestNestedItem
default_item_class = NestedItem
class TestItemLoader(NameItemLoader):
class ProcessorItemLoader(NameItemLoader):
name_in = MapCompose(lambda v: v.title())
@ -68,11 +68,11 @@ def processor_with_args(value, other=None, loader_context=None):
class BasicItemLoaderTest(unittest.TestCase):
def test_add_value_on_unknown_field(self):
il = TestItemLoader()
il = ProcessorItemLoader()
self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"])
def test_load_item_using_default_loader(self):
i = TestItem()
i = SummaryItem()
i["summary"] = "lala"
il = ItemLoader(item=i)
il.add_value("name", "marta")
@ -82,7 +82,7 @@ class BasicItemLoaderTest(unittest.TestCase):
self.assertEqual(item["name"], ["marta"])
def test_load_item_using_custom_loader(self):
il = TestItemLoader()
il = ProcessorItemLoader()
il.add_value("name", "marta")
item = il.load_item()
self.assertEqual(item["name"], ["Marta"])
@ -194,7 +194,7 @@ class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase
class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase):
item_class = TestDataClass
item_class = NameDataClass
class BaseNoInputReprocessingLoader(ItemLoader):
@ -289,11 +289,11 @@ class SelectortemLoaderTest(unittest.TestCase):
)
def test_init_method(self):
l = TestItemLoader()
l = ProcessorItemLoader()
self.assertEqual(l.selector, None)
def test_init_method_errors(self):
l = TestItemLoader()
l = ProcessorItemLoader()
self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href")
self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href")
self.assertRaises(RuntimeError, l.get_xpath, "//a/@href")
@ -303,7 +303,7 @@ class SelectortemLoaderTest(unittest.TestCase):
def test_init_method_with_selector(self):
sel = Selector(text="<html><body><div>marta</div></body></html>")
l = TestItemLoader(selector=sel)
l = ProcessorItemLoader(selector=sel)
self.assertIs(l.selector, sel)
l.add_xpath("name", "//div/text()")
@ -311,7 +311,7 @@ class SelectortemLoaderTest(unittest.TestCase):
def test_init_method_with_selector_css(self):
sel = Selector(text="<html><body><div>marta</div></body></html>")
l = TestItemLoader(selector=sel)
l = ProcessorItemLoader(selector=sel)
self.assertIs(l.selector, sel)
l.add_css("name", "div::text")
@ -320,18 +320,18 @@ class SelectortemLoaderTest(unittest.TestCase):
def test_init_method_with_base_response(self):
"""Selector should be None after initialization"""
response = Response("https://scrapy.org")
l = TestItemLoader(response=response)
l = ProcessorItemLoader(response=response)
self.assertIs(l.selector, None)
def test_init_method_with_response(self):
l = TestItemLoader(response=self.response)
l = ProcessorItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_xpath("name", "//div/text()")
self.assertEqual(l.get_output_value("name"), ["Marta"])
def test_init_method_with_response_css(self):
l = TestItemLoader(response=self.response)
l = ProcessorItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_css("name", "div::text")
@ -350,12 +350,12 @@ class SelectortemLoaderTest(unittest.TestCase):
)
def test_add_xpath_re(self):
l = TestItemLoader(response=self.response)
l = ProcessorItemLoader(response=self.response)
l.add_xpath("name", "//div/text()", re="ma")
self.assertEqual(l.get_output_value("name"), ["Ma"])
def test_replace_xpath(self):
l = TestItemLoader(response=self.response)
l = ProcessorItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_xpath("name", "//div/text()")
self.assertEqual(l.get_output_value("name"), ["Marta"])
@ -366,7 +366,7 @@ class SelectortemLoaderTest(unittest.TestCase):
self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"])
def test_get_xpath(self):
l = TestItemLoader(response=self.response)
l = ProcessorItemLoader(response=self.response)
self.assertEqual(l.get_xpath("//p/text()"), ["paragraph"])
self.assertEqual(l.get_xpath("//p/text()", TakeFirst()), "paragraph")
self.assertEqual(l.get_xpath("//p/text()", TakeFirst(), re="pa"), "pa")
@ -376,14 +376,14 @@ class SelectortemLoaderTest(unittest.TestCase):
)
def test_replace_xpath_multi_fields(self):
l = TestItemLoader(response=self.response)
l = ProcessorItemLoader(response=self.response)
l.add_xpath(None, "//div/text()", TakeFirst(), lambda x: {"name": x})
self.assertEqual(l.get_output_value("name"), ["Marta"])
l.replace_xpath(None, "//p/text()", TakeFirst(), lambda x: {"name": x})
self.assertEqual(l.get_output_value("name"), ["Paragraph"])
def test_replace_xpath_re(self):
l = TestItemLoader(response=self.response)
l = ProcessorItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_xpath("name", "//div/text()")
self.assertEqual(l.get_output_value("name"), ["Marta"])
@ -391,7 +391,7 @@ class SelectortemLoaderTest(unittest.TestCase):
self.assertEqual(l.get_output_value("name"), ["Ma"])
def test_add_css_re(self):
l = TestItemLoader(response=self.response)
l = ProcessorItemLoader(response=self.response)
l.add_css("name", "div::text", re="ma")
self.assertEqual(l.get_output_value("name"), ["Ma"])
@ -399,7 +399,7 @@ class SelectortemLoaderTest(unittest.TestCase):
self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"])
def test_replace_css(self):
l = TestItemLoader(response=self.response)
l = ProcessorItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_css("name", "div::text")
self.assertEqual(l.get_output_value("name"), ["Marta"])
@ -415,7 +415,7 @@ class SelectortemLoaderTest(unittest.TestCase):
self.assertEqual(l.get_output_value("url"), ["/images/logo.png"])
def test_get_css(self):
l = TestItemLoader(response=self.response)
l = ProcessorItemLoader(response=self.response)
self.assertEqual(l.get_css("p::text"), ["paragraph"])
self.assertEqual(l.get_css("p::text", TakeFirst()), "paragraph")
self.assertEqual(l.get_css("p::text", TakeFirst(), re="pa"), "pa")
@ -427,7 +427,7 @@ class SelectortemLoaderTest(unittest.TestCase):
)
def test_replace_css_multi_fields(self):
l = TestItemLoader(response=self.response)
l = ProcessorItemLoader(response=self.response)
l.add_css(None, "div::text", TakeFirst(), lambda x: {"name": x})
self.assertEqual(l.get_output_value("name"), ["Marta"])
l.replace_css(None, "p::text", TakeFirst(), lambda x: {"name": x})
@ -439,7 +439,7 @@ class SelectortemLoaderTest(unittest.TestCase):
self.assertEqual(l.get_output_value("url"), ["/images/logo.png"])
def test_replace_css_re(self):
l = TestItemLoader(response=self.response)
l = ProcessorItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_css("url", "a::attr(href)")
self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])

46
tests/test_loader_deprecated.py
View File

@ -24,17 +24,17 @@ class NameItem(Item):
name = Field()
class TestItem(NameItem):
class SummaryItem(NameItem):
url = Field()
summary = Field()
# test item loaders
class NameItemLoader(ItemLoader):
default_item_class = TestItem
default_item_class = SummaryItem
class TestItemLoader(NameItemLoader):
class ProcessorItemLoader(NameItemLoader):
name_in = MapCompose(lambda v: v.title())
@ -51,7 +51,7 @@ def processor_with_args(value, other=None, loader_context=None):
class BasicItemLoaderTest(unittest.TestCase):
def test_load_item_using_default_loader(self):
i = TestItem()
i = SummaryItem()
i["summary"] = "lala"
il = ItemLoader(item=i)
il.add_value("name", "marta")
@ -61,7 +61,7 @@ class BasicItemLoaderTest(unittest.TestCase):
self.assertEqual(item["name"], ["marta"])
def test_load_item_using_custom_loader(self):
il = TestItemLoader()
il = ProcessorItemLoader()
il.add_value("name", "marta")
item = il.load_item()
self.assertEqual(item["name"], ["Marta"])
@ -125,7 +125,7 @@ class BasicItemLoaderTest(unittest.TestCase):
)
def test_add_value(self):
il = TestItemLoader()
il = ProcessorItemLoader()
il.add_value("name", "marta")
self.assertEqual(il.get_collected_values("name"), ["Marta"])
self.assertEqual(il.get_output_value("name"), ["Marta"])
@ -146,7 +146,7 @@ class BasicItemLoaderTest(unittest.TestCase):
self.assertEqual(il.get_collected_values("name"), [0])
def test_replace_value(self):
il = TestItemLoader()
il = ProcessorItemLoader()
il.replace_value("name", "marta")
self.assertEqual(il.get_collected_values("name"), ["Marta"])
self.assertEqual(il.get_output_value("name"), ["Marta"])
@ -229,7 +229,7 @@ class BasicItemLoaderTest(unittest.TestCase):
self.assertEqual(il.get_output_value("name"), ["mart"])
def test_input_processor_inheritance(self):
class ChildItemLoader(TestItemLoader):
class ChildItemLoader(ProcessorItemLoader):
url_in = MapCompose(lambda v: v.lower())
il = ChildItemLoader()
@ -265,8 +265,8 @@ class BasicItemLoaderTest(unittest.TestCase):
self.assertEqual(il.get_output_value("name"), ["marta"])
def test_extend_custom_input_processors(self):
class ChildItemLoader(TestItemLoader):
name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
class ChildItemLoader(ProcessorItemLoader):
name_in = MapCompose(ProcessorItemLoader.name_in, str.swapcase)
il = ChildItemLoader()
il.add_value("name", "marta")
@ -283,11 +283,11 @@ class BasicItemLoaderTest(unittest.TestCase):
self.assertEqual(il.get_output_value("name"), ["MART"])
def test_output_processor_using_function(self):
il = TestItemLoader()
il = ProcessorItemLoader()
il.add_value("name", ["mar", "ta"])
self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
class TakeFirstItemLoader(TestItemLoader):
class TakeFirstItemLoader(ProcessorItemLoader):
name_out = " ".join
il = TakeFirstItemLoader()
@ -296,7 +296,7 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_output_processor_error(self):
class TestItemLoader(ItemLoader):
default_item_class = TestItem
default_item_class = SummaryItem
name_out = MapCompose(float)
il = TestItemLoader()
@ -319,18 +319,18 @@ class BasicItemLoaderTest(unittest.TestCase):
assert expected_exc_str in s, s
def test_output_processor_using_classes(self):
il = TestItemLoader()
il = ProcessorItemLoader()
il.add_value("name", ["mar", "ta"])
self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
class TakeFirstItemLoader(TestItemLoader):
class TakeFirstItemLoader(ProcessorItemLoader):
name_out = Join()
il = TakeFirstItemLoader()
il.add_value("name", ["mar", "ta"])
self.assertEqual(il.get_output_value("name"), "Mar Ta")
class TakeFirstItemLoader2(TestItemLoader):
class TakeFirstItemLoader2(ProcessorItemLoader):
name_out = Join("<br>")
il = TakeFirstItemLoader2()
@ -338,11 +338,11 @@ class BasicItemLoaderTest(unittest.TestCase):
self.assertEqual(il.get_output_value("name"), "Mar<br>Ta")
def test_default_output_processor(self):
il = TestItemLoader()
il = ProcessorItemLoader()
il.add_value("name", ["mar", "ta"])
self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
class LalaItemLoader(TestItemLoader):
class LalaItemLoader(ProcessorItemLoader):
default_output_processor = Identity()
il = LalaItemLoader()
@ -350,7 +350,7 @@ class BasicItemLoaderTest(unittest.TestCase):
self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
def test_loader_context_on_declaration(self):
class ChildItemLoader(TestItemLoader):
class ChildItemLoader(ProcessorItemLoader):
url_in = MapCompose(processor_with_args, key="val")
il = ChildItemLoader()
@ -360,7 +360,7 @@ class BasicItemLoaderTest(unittest.TestCase):
self.assertEqual(il.get_output_value("url"), ["val"])
def test_loader_context_on_instantiation(self):
class ChildItemLoader(TestItemLoader):
class ChildItemLoader(ProcessorItemLoader):
url_in = MapCompose(processor_with_args)
il = ChildItemLoader(key="val")
@ -370,7 +370,7 @@ class BasicItemLoaderTest(unittest.TestCase):
self.assertEqual(il.get_output_value("url"), ["val"])
def test_loader_context_on_assign(self):
class ChildItemLoader(TestItemLoader):
class ChildItemLoader(ProcessorItemLoader):
url_in = MapCompose(processor_with_args)
il = ChildItemLoader()
@ -384,10 +384,10 @@ class BasicItemLoaderTest(unittest.TestCase):
def processor(value, loader_context):
return loader_context["item"]["name"]
class ChildItemLoader(TestItemLoader):
class ChildItemLoader(ProcessorItemLoader):
url_in = MapCompose(processor)
it = TestItem(name="marta")
it = SummaryItem(name="marta")
il = ChildItemLoader(item=it)
il.add_value("url", "text")
self.assertEqual(il.get_output_value("url"), ["marta"])

8
tests/test_middleware.py
View File

@ -40,7 +40,7 @@ class MOff:
raise NotConfigured("foo")
class TestMiddlewareManager(MiddlewareManager):
class MyMiddlewareManager(MiddlewareManager):
@classmethod
def _get_mwlist_from_settings(cls, settings):
return [M1, MOff, M3]
@ -54,7 +54,7 @@ class TestMiddlewareManager(MiddlewareManager):
class MiddlewareManagerTest(unittest.TestCase):
def test_init(self):
m1, m2, m3 = M1(), M2(), M3()
mwman = TestMiddlewareManager(m1, m2, m3)
mwman = MyMiddlewareManager(m1, m2, m3)
self.assertEqual(
list(mwman.methods["open_spider"]), [m1.open_spider, m2.open_spider]
)
@ -64,7 +64,7 @@ class MiddlewareManagerTest(unittest.TestCase):
self.assertEqual(list(mwman.methods["process"]), [m1.process, m3.process])
def test_methods(self):
mwman = TestMiddlewareManager(M1(), M2(), M3())
mwman = MyMiddlewareManager(M1(), M2(), M3())
self.assertEqual(
[x.__self__.__class__ for x in mwman.methods["open_spider"]], [M1, M2]
)
@ -82,6 +82,6 @@ class MiddlewareManagerTest(unittest.TestCase):
def test_enabled_from_settings(self):
crawler = get_crawler()
mwman = TestMiddlewareManager.from_crawler(crawler)
mwman = MyMiddlewareManager.from_crawler(crawler)
classes = [x.__class__ for x in mwman.middlewares]
self.assertEqual(classes, [M1, M3])

6
tests/test_pipelines.py
View File

@ -1,6 +1,6 @@
import asyncio
from pytest import mark
import pytest
from twisted.internet import defer
from twisted.internet.defer import Deferred
from twisted.trial import unittest
@ -118,14 +118,14 @@ class PipelineTestCase(unittest.TestCase):
yield crawler.crawl(mockserver=self.mockserver)
self.assertEqual(len(self.items), 1)
@mark.only_asyncio()
@pytest.mark.only_asyncio
@defer.inlineCallbacks
def test_asyncdef_asyncio_pipeline(self):
crawler = self._create_crawler(AsyncDefAsyncioPipeline)
yield crawler.crawl(mockserver=self.mockserver)
self.assertEqual(len(self.items), 1)
@mark.only_not_asyncio()
@pytest.mark.only_not_asyncio
@defer.inlineCallbacks
def test_asyncdef_not_asyncio_pipeline(self):
crawler = self._create_crawler(AsyncDefNotAsyncioPipeline)

28
tests/test_request_dict.py
View File

@ -11,7 +11,7 @@ class CustomRequest(Request):
class RequestSerializationTest(unittest.TestCase):
def setUp(self):
self.spider = TestSpider()
self.spider = MethodsSpider()
def test_basic(self):
r = Request("http://www.example.com")
@ -96,18 +96,22 @@ class RequestSerializationTest(unittest.TestCase):
def test_private_reference_callback_serialization(self):
r = Request(
"http://www.example.com",
callback=self.spider._TestSpider__parse_item_reference,
errback=self.spider._TestSpider__handle_error_reference,
callback=self.spider._MethodsSpider__parse_item_reference,
errback=self.spider._MethodsSpider__handle_error_reference,
)
self._assert_serializes_ok(r, spider=self.spider)
request_dict = r.to_dict(spider=self.spider)
self.assertEqual(request_dict["callback"], "_TestSpider__parse_item_reference")
self.assertEqual(request_dict["errback"], "_TestSpider__handle_error_reference")
self.assertEqual(
request_dict["callback"], "_MethodsSpider__parse_item_reference"
)
self.assertEqual(
request_dict["errback"], "_MethodsSpider__handle_error_reference"
)
def test_private_callback_serialization(self):
r = Request(
"http://www.example.com",
callback=self.spider._TestSpider__parse_item_private,
callback=self.spider._MethodsSpider__parse_item_private,
errback=self.spider.handle_error,
)
self._assert_serializes_ok(r, spider=self.spider)
@ -115,7 +119,7 @@ class RequestSerializationTest(unittest.TestCase):
def test_mixin_private_callback_serialization(self):
r = Request(
"http://www.example.com",
callback=self.spider._TestSpiderMixin__mixin_callback,
callback=self.spider._SpiderMixin__mixin_callback,
errback=self.spider.handle_error,
)
self._assert_serializes_ok(r, spider=self.spider)
@ -152,18 +156,18 @@ class RequestSerializationTest(unittest.TestCase):
def test_callback_not_available(self):
"""Callback method is not available in the spider passed to from_dict"""
spider = TestSpiderDelegation()
spider = SpiderDelegation()
r = Request("http://www.example.com", callback=spider.delegated_callback)
d = r.to_dict(spider=spider)
self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo"))
class TestSpiderMixin:
class SpiderMixin:
def __mixin_callback(self, response): # pylint: disable=unused-private-member
pass
class TestSpiderDelegation:
class SpiderDelegation:
def delegated_callback(self, response):
pass
@ -184,7 +188,7 @@ def private_handle_error(failure):
pass
class TestSpider(Spider, TestSpiderMixin):
class MethodsSpider(Spider, SpiderMixin):
name = "test"
parse_item_reference = parse_item
handle_error_reference = handle_error
@ -193,7 +197,7 @@ class TestSpider(Spider, TestSpiderMixin):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.delegated_callback = TestSpiderDelegation().delegated_callback
self.delegated_callback = SpiderDelegation().delegated_callback
def parse_item(self, response):
pass

6
tests/test_scheduler_base.py
View File

@ -51,8 +51,8 @@ class SimpleScheduler(MinimalScheduler):
return len(self.requests)
class TestSpider(Spider):
name = "test"
class PathsSpider(Spider):
name = "paths"
def __init__(self, mockserver, *args, **kwargs):
super().__init__(*args, **kwargs)
@ -155,7 +155,7 @@ class MinimalSchedulerCrawlTest(TwistedTestCase):
"SCHEDULER": self.scheduler_cls,
}
with LogCapture() as log:
crawler = get_crawler(TestSpider, settings)
crawler = get_crawler(PathsSpider, settings)
yield crawler.crawl(mockserver)
for path in PATHS:
self.assertIn(f"{{'path': '{path}'}}", str(log))

4
tests/test_signals.py
View File

@ -1,4 +1,4 @@
from pytest import mark
import pytest
from twisted.internet import defer
from twisted.trial import unittest
@ -37,7 +37,7 @@ class AsyncSignalTestCase(unittest.TestCase):
item = await get_from_asyncio_queue(item)
self.items.append(item)
@mark.only_asyncio()
@pytest.mark.only_asyncio
@defer.inlineCallbacks
def test_simple_pipeline(self):
crawler = get_crawler(ItemSpider)

26
tests/test_squeues.py
View File

@ -15,7 +15,7 @@ from scrapy.squeues import (
)
class TestItem(Item):
class MyItem(Item):
name = Field()
@ -23,8 +23,8 @@ def _test_procesor(x):
return x + x
class TestLoader(ItemLoader):
default_item_class = TestItem
class MyLoader(ItemLoader):
default_item_class = MyItem
name_out = staticmethod(_test_procesor)
@ -80,19 +80,19 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
def test_serialize_item(self):
q = self.queue()
i = TestItem(name="foo")
i = MyItem(name="foo")
q.push(i)
i2 = q.pop()
assert isinstance(i2, TestItem)
assert isinstance(i2, MyItem)
self.assertEqual(i, i2)
def test_serialize_loader(self):
q = self.queue()
loader = TestLoader()
loader = MyLoader()
q.push(loader)
loader2 = q.pop()
assert isinstance(loader2, TestLoader)
assert loader2.default_item_class is TestItem
assert isinstance(loader2, MyLoader)
assert loader2.default_item_class is MyItem
self.assertEqual(loader2.name_out("x"), "xx")
def test_serialize_request_recursive(self):
@ -161,19 +161,19 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
def test_serialize_item(self):
q = self.queue()
i = TestItem(name="foo")
i = MyItem(name="foo")
q.push(i)
i2 = q.pop()
assert isinstance(i2, TestItem)
assert isinstance(i2, MyItem)
self.assertEqual(i, i2)
def test_serialize_loader(self):
q = self.queue()
loader = TestLoader()
loader = MyLoader()
q.push(loader)
loader2 = q.pop()
assert isinstance(loader2, TestLoader)
assert loader2.default_item_class is TestItem
assert isinstance(loader2, MyLoader)
assert loader2.default_item_class is MyItem
self.assertEqual(loader2.name_out("x"), "xx")
def test_serialize_request_recursive(self):

6
tests/test_utils_asyncio.py
View File

@ -1,7 +1,7 @@
import asyncio
import warnings
from pytest import mark
import pytest
from twisted.trial.unittest import TestCase
from scrapy.utils.defer import deferred_f_from_coro_f
@ -12,7 +12,7 @@ from scrapy.utils.reactor import (
)
@mark.usefixtures("reactor_pytest")
@pytest.mark.usefixtures("reactor_pytest")
class AsyncioTest(TestCase):
def test_is_asyncio_reactor_installed(self):
# the result should depend only on the pytest --reactor argument
@ -30,7 +30,7 @@ class AsyncioTest(TestCase):
assert original_reactor == reactor
@mark.only_asyncio()
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
async def test_set_asyncio_event_loop(self):
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")

2
tests/test_utils_datatypes.py
View File

@ -16,8 +16,6 @@ from scrapy.utils.datatypes import (
)
from scrapy.utils.python import garbage_collect
__doctests__ = ["scrapy.utils.datatypes"]
class CaseInsensitiveDictMixin:
def test_init_dict(self):

4
tests/test_utils_defer.py
View File

@ -1,6 +1,6 @@
import random
from pytest import mark
import pytest
from twisted.internet import defer, reactor
from twisted.python.failure import Failure
from twisted.trial import unittest
@ -150,7 +150,7 @@ class AsyncDefTestsuiteTest(unittest.TestCase):
async def test_deferred_f_from_coro_f_generator(self):
yield
@mark.xfail(reason="Checks that the test is actually executed", strict=True)
@pytest.mark.xfail(reason="Checks that the test is actually executed", strict=True)
@deferred_f_from_coro_f
async def test_deferred_f_from_coro_f_xfail(self):
raise RuntimeError("This is expected to be raised")

4
tests/test_utils_log.py
View File

@ -119,7 +119,7 @@ class StreamLoggerTest(unittest.TestCase):
@pytest.mark.parametrize(
("base_extra", "log_extra", "expected_extra"),
(
[
(
{"spider": "test"},
{"extra": {"log_extra": "info"}},
@ -135,7 +135,7 @@ class StreamLoggerTest(unittest.TestCase):
{"extra": {"spider": "test2"}},
{"extra": {"spider": "test"}},
),
),
],
)
def test_spider_logger_adapter_process(
base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict

2
tests/test_utils_misc/__init__.py
View File

@ -17,8 +17,6 @@ from scrapy.utils.misc import (
walk_modules,
)
__doctests__ = ["scrapy.utils.misc"]
class UtilsMiscTestCase(unittest.TestCase):
def test_load_object_class(self):

2
tests/test_utils_python.py
View File

@ -20,8 +20,6 @@ from scrapy.utils.python import (
without_none_values,
)
__doctests__ = ["scrapy.utils.python"]
class MutableChainTest(unittest.TestCase):
def test_mutablechain(self):

8
tests/test_utils_response.py
View File

@ -15,8 +15,6 @@ from scrapy.utils.response import (
response_status_message,
)
__doctests__ = ["scrapy.utils.response"]
class ResponseUtilsTest(unittest.TestCase):
dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response")
@ -207,8 +205,8 @@ class ResponseUtilsTest(unittest.TestCase):
@pytest.mark.parametrize(
"input_body,output_body",
(
("input_body", "output_body"),
[
(
b"a<!--",
b"a",
@ -237,7 +235,7 @@ class ResponseUtilsTest(unittest.TestCase):
b"a<!--b--><!--c-->d",
b"ad",
),
),
],
)
def test_remove_html_comments(input_body, output_body):
assert _remove_html_comments(input_body) == output_body, (

6
tests/test_utils_signal.py
View File

@ -1,7 +1,7 @@
import asyncio
import pytest
from pydispatch import dispatcher
from pytest import mark
from testfixtures import LogCapture
from twisted.internet import defer, reactor
from twisted.python.failure import Failure
@ -67,7 +67,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest):
return d
@mark.usefixtures("reactor_pytest")
@pytest.mark.usefixtures("reactor_pytest")
class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
async def ok_handler(self, arg, handlers_called):
handlers_called.add(self.ok_handler)
@ -76,7 +76,7 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
return "OK"
@mark.only_asyncio()
@pytest.mark.only_asyncio
class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
async def ok_handler(self, arg, handlers_called):
handlers_called.add(self.ok_handler)

2
tests/test_utils_template.py
View File

@ -5,8 +5,6 @@ from tempfile import mkdtemp
from scrapy.utils.template import render_templatefile
__doctests__ = ["scrapy.utils.template"]
class UtilsRenderTemplateFileTestCase(unittest.TestCase):
def setUp(self):

2
tests/test_utils_url.py
View File

@ -17,8 +17,6 @@ from scrapy.utils.url import ( # type: ignore[attr-defined]
url_is_from_spider,
)
__doctests__ = ["scrapy.utils.url"]
class UrlUtilsTest(unittest.TestCase):
def test_url_is_from_any_domain(self):