RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Convert remaining unittest assert* calls, use the tmp_path fixture. (#6725)

This commit is contained in:
Andrey Rakhmatullin 2025-03-11 21:00:36 +04:00 committed by GitHub
parent 803b4f258d
commit eb654aa1a8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 62 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pyproject.toml
View File

@ -374,10 +374,6 @@ ignore = [
"B904",
# Use capitalized environment variable
"SIM112",
# Temporarily silenced PT rules
# Use a regular `assert` instead of unittest-style `assertEqual`
"PT009",
]
[tool.ruff.lint.per-file-ignores]

2
scrapy/utils/test.py
View File

@ -156,7 +156,7 @@ def assert_samelines(
category=ScrapyDeprecationWarning,
stacklevel=2,
)
testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) # noqa: PT009
def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:

13
tests/test_downloadermiddleware_robotstxt.py
View File

@ -1,3 +1,4 @@
from typing import Any
from unittest import mock
import pytest
@ -171,7 +172,11 @@ Disallow: /some/randome/page.html
middleware = RobotsTxtMiddleware(self.crawler)
middleware._logerror = mock.MagicMock(side_effect=middleware._logerror)
deferred = middleware.process_request(Request("http://site.local"), None)
deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called))
def check_called(_: Any) -> None:
assert middleware._logerror.called
deferred.addCallback(check_called)
return deferred
def test_robotstxt_immediate_error(self):
@ -202,7 +207,11 @@ Disallow: /some/randome/page.html
mw_module_logger.error = mock.MagicMock()
d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called))
def check_not_called(_: Any) -> None:
assert not mw_module_logger.error.called # type: ignore[attr-defined]
d.addCallback(check_not_called)
return d
def test_robotstxt_user_agent_setting(self):

8
tests/test_engine.py
View File

@ -433,10 +433,12 @@ class TestEngine(TestEngineBase):
e = ExecutionEngine(get_crawler(MySpider), lambda _: None)
yield e.open_spider(MySpider(), [])
e.start()
def cb(exc: BaseException) -> None:
assert str(exc), "Engine already running"
try:
yield self.assertFailure(e.start(), RuntimeError).addBoth(
lambda exc: self.assertEqual(str(exc), "Engine already running")
)
yield self.assertFailure(e.start(), RuntimeError).addBoth(cb)
finally:
yield e.stop()

58
tests/test_pipeline_files.py
View File

@ -266,17 +266,11 @@ class TestFilesPipeline(unittest.TestCase):
class FilesPipelineTestCaseFieldsMixin:
def setup_method(self):
self.tempdir = mkdtemp()
def teardown_method(self):
rmtree(self.tempdir)
def test_item_fields_default(self):
def test_item_fields_default(self, tmp_path):
url = "http://www.example.com/files/1.txt"
item = self.item_class(name="item1", file_urls=[url])
pipeline = FilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": self.tempdir})
get_crawler(None, {"FILES_STORE": tmp_path})
)
requests = list(pipeline.get_media_requests(item, None))
assert requests[0].url == url
@ -286,14 +280,14 @@ class FilesPipelineTestCaseFieldsMixin:
assert files == [results[0][1]]
assert isinstance(item, self.item_class)
def test_item_fields_override_settings(self):
def test_item_fields_override_settings(self, tmp_path):
url = "http://www.example.com/files/1.txt"
item = self.item_class(name="item1", custom_file_urls=[url])
pipeline = FilesPipeline.from_crawler(
get_crawler(
None,
{
"FILES_STORE": self.tempdir,
"FILES_STORE": tmp_path,
"FILES_URLS_FIELD": "custom_file_urls",
"FILES_RESULT_FIELD": "custom_files",
},
@ -368,13 +362,7 @@ class TestFilesPipelineCustomSettings:
("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"),
}
def setup_method(self):
self.tempdir = mkdtemp()
def teardown_method(self):
rmtree(self.tempdir)
def _generate_fake_settings(self, prefix=None):
def _generate_fake_settings(self, tmp_path, prefix=None):
def random_string():
return "".join([chr(random.randint(97, 123)) for _ in range(10)])
@ -382,7 +370,7 @@ class TestFilesPipelineCustomSettings:
"FILES_EXPIRES": random.randint(100, 1000),
"FILES_URLS_FIELD": random_string(),
"FILES_RESULT_FIELD": random_string(),
"FILES_STORE": self.tempdir,
"FILES_STORE": tmp_path,
}
if not prefix:
return settings
@ -400,16 +388,16 @@ class TestFilesPipelineCustomSettings:
return UserDefinedFilePipeline
def test_different_settings_for_different_instances(self):
def test_different_settings_for_different_instances(self, tmp_path):
"""
If there are different instances with different settings they should keep
different settings.
"""
custom_settings = self._generate_fake_settings()
custom_settings = self._generate_fake_settings(tmp_path)
another_pipeline = FilesPipeline.from_crawler(
get_crawler(None, custom_settings)
)
one_pipeline = FilesPipeline(self.tempdir, crawler=get_crawler(None))
one_pipeline = FilesPipeline(tmp_path, crawler=get_crawler(None))
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
default_value = self.default_cls_settings[pipe_attr]
assert getattr(one_pipeline, pipe_attr) == default_value
@ -417,24 +405,24 @@ class TestFilesPipelineCustomSettings:
assert default_value != custom_value
assert getattr(another_pipeline, pipe_ins_attr) == custom_value
def test_subclass_attributes_preserved_if_no_settings(self):
def test_subclass_attributes_preserved_if_no_settings(self, tmp_path):
"""
If subclasses override class attributes and there are no special settings those values should be kept.
"""
pipe_cls = self._generate_fake_pipeline()
pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": self.tempdir}))
pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": tmp_path}))
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
custom_value = getattr(pipe, pipe_ins_attr)
assert custom_value != self.default_cls_settings[pipe_attr]
assert getattr(pipe, pipe_ins_attr) == getattr(pipe, pipe_attr)
def test_subclass_attrs_preserved_custom_settings(self):
def test_subclass_attrs_preserved_custom_settings(self, tmp_path):
"""
If file settings are defined but they are not defined for subclass
settings should be preserved.
"""
pipeline_cls = self._generate_fake_pipeline()
settings = self._generate_fake_settings()
settings = self._generate_fake_settings(tmp_path)
pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
value = getattr(pipeline, pipe_ins_attr)
@ -442,7 +430,7 @@ class TestFilesPipelineCustomSettings:
assert value != self.default_cls_settings[pipe_attr]
assert value == setting_value
def test_no_custom_settings_for_subclasses(self):
def test_no_custom_settings_for_subclasses(self, tmp_path):
"""
If there are no settings for subclass and no subclass attributes, pipeline should use
attributes of base class.
@ -452,14 +440,14 @@ class TestFilesPipelineCustomSettings:
pass
user_pipeline = UserDefinedFilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": self.tempdir})
get_crawler(None, {"FILES_STORE": tmp_path})
)
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
# Values from settings for custom pipeline should be set on pipeline instance.
custom_value = self.default_cls_settings.get(pipe_attr.upper())
assert getattr(user_pipeline, pipe_ins_attr) == custom_value
def test_custom_settings_for_subclasses(self):
def test_custom_settings_for_subclasses(self, tmp_path):
"""
If there are custom settings for subclass and NO class attributes, pipeline should use custom
settings.
@ -469,7 +457,7 @@ class TestFilesPipelineCustomSettings:
pass
prefix = UserDefinedFilesPipeline.__name__.upper()
settings = self._generate_fake_settings(prefix=prefix)
settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = UserDefinedFilesPipeline.from_crawler(
get_crawler(None, settings)
)
@ -479,14 +467,14 @@ class TestFilesPipelineCustomSettings:
assert custom_value != self.default_cls_settings[pipe_attr]
assert getattr(user_pipeline, pipe_inst_attr) == custom_value
def test_custom_settings_and_class_attrs_for_subclasses(self):
def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path):
"""
If there are custom settings for subclass AND class attributes
setting keys are preferred and override attributes.
"""
pipeline_cls = self._generate_fake_pipeline()
prefix = pipeline_cls.__name__.upper()
settings = self._generate_fake_settings(prefix=prefix)
settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
for (
pipe_cls_attr,
@ -497,13 +485,13 @@ class TestFilesPipelineCustomSettings:
assert custom_value != self.default_cls_settings[pipe_cls_attr]
assert getattr(user_pipeline, pipe_inst_attr) == custom_value
def test_cls_attrs_with_DEFAULT_prefix(self):
def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path):
class UserDefinedFilesPipeline(FilesPipeline):
DEFAULT_FILES_RESULT_FIELD = "this"
DEFAULT_FILES_URLS_FIELD = "that"
pipeline = UserDefinedFilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": self.tempdir})
get_crawler(None, {"FILES_STORE": tmp_path})
)
assert (
pipeline.files_result_field
@ -514,12 +502,12 @@ class TestFilesPipelineCustomSettings:
== UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD
)
def test_user_defined_subclass_default_key_names(self):
def test_user_defined_subclass_default_key_names(self, tmp_path):
"""Test situation when user defines subclass of FilesPipeline,
but uses attribute names for default pipeline (without prefixing
them with pipeline class name).
"""
settings = self._generate_fake_settings()
settings = self._generate_fake_settings(tmp_path)
class UserPipe(FilesPipeline):
pass

44
tests/test_pipeline_images.py
View File

@ -314,13 +314,7 @@ class TestImagesPipelineCustomSettings:
"IMAGES_RESULT_FIELD": "images",
}
def setup_method(self):
self.tempdir = mkdtemp()
def teardown_method(self):
rmtree(self.tempdir)
def _generate_fake_settings(self, prefix=None):
def _generate_fake_settings(self, tmp_path, prefix=None):
"""
:param prefix: string for setting keys
:return: dictionary of image pipeline settings
@ -331,7 +325,7 @@ class TestImagesPipelineCustomSettings:
settings = {
"IMAGES_EXPIRES": random.randint(100, 1000),
"IMAGES_STORE": self.tempdir,
"IMAGES_STORE": tmp_path,
"IMAGES_RESULT_FIELD": random_string(),
"IMAGES_URLS_FIELD": random_string(),
"IMAGES_MIN_WIDTH": random.randint(1, 1000),
@ -368,13 +362,13 @@ class TestImagesPipelineCustomSettings:
return UserDefinedImagePipeline
def test_different_settings_for_different_instances(self):
def test_different_settings_for_different_instances(self, tmp_path):
"""
If there are two instances of ImagesPipeline class with different settings, they should
have different settings.
"""
custom_settings = self._generate_fake_settings()
default_sts_pipe = ImagesPipeline(self.tempdir, crawler=get_crawler(None))
custom_settings = self._generate_fake_settings(tmp_path)
default_sts_pipe = ImagesPipeline(tmp_path, crawler=get_crawler(None))
user_sts_pipe = ImagesPipeline.from_crawler(get_crawler(None, custom_settings))
for pipe_attr, settings_attr in self.img_cls_attribute_names:
expected_default_value = self.default_pipeline_settings.get(pipe_attr)
@ -385,14 +379,14 @@ class TestImagesPipelineCustomSettings:
)
assert getattr(user_sts_pipe, pipe_attr.lower()) == custom_value
def test_subclass_attrs_preserved_default_settings(self):
def test_subclass_attrs_preserved_default_settings(self, tmp_path):
"""
If image settings are not defined at all subclass of ImagePipeline takes values
from class attributes.
"""
pipeline_cls = self._generate_fake_pipeline_subclass()
pipeline = pipeline_cls.from_crawler(
get_crawler(None, {"IMAGES_STORE": self.tempdir})
get_crawler(None, {"IMAGES_STORE": tmp_path})
)
for pipe_attr, settings_attr in self.img_cls_attribute_names:
# Instance attribute (lowercase) must be equal to class attribute (uppercase).
@ -400,13 +394,13 @@ class TestImagesPipelineCustomSettings:
assert attr_value != self.default_pipeline_settings[pipe_attr]
assert attr_value == getattr(pipeline, pipe_attr)
def test_subclass_attrs_preserved_custom_settings(self):
def test_subclass_attrs_preserved_custom_settings(self, tmp_path):
"""
If image settings are defined but they are not defined for subclass default
values taken from settings should be preserved.
"""
pipeline_cls = self._generate_fake_pipeline_subclass()
settings = self._generate_fake_settings()
settings = self._generate_fake_settings(tmp_path)
pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
for pipe_attr, settings_attr in self.img_cls_attribute_names:
# Instance attribute (lowercase) must be equal to
@ -416,7 +410,7 @@ class TestImagesPipelineCustomSettings:
setings_value = settings.get(settings_attr)
assert value == setings_value
def test_no_custom_settings_for_subclasses(self):
def test_no_custom_settings_for_subclasses(self, tmp_path):
"""
If there are no settings for subclass and no subclass attributes, pipeline should use
attributes of base class.
@ -426,14 +420,14 @@ class TestImagesPipelineCustomSettings:
pass
user_pipeline = UserDefinedImagePipeline.from_crawler(
get_crawler(None, {"IMAGES_STORE": self.tempdir})
get_crawler(None, {"IMAGES_STORE": tmp_path})
)
for pipe_attr, settings_attr in self.img_cls_attribute_names:
# Values from settings for custom pipeline should be set on pipeline instance.
custom_value = self.default_pipeline_settings.get(pipe_attr.upper())
assert getattr(user_pipeline, pipe_attr.lower()) == custom_value
def test_custom_settings_for_subclasses(self):
def test_custom_settings_for_subclasses(self, tmp_path):
"""
If there are custom settings for subclass and NO class attributes, pipeline should use custom
settings.
@ -443,7 +437,7 @@ class TestImagesPipelineCustomSettings:
pass
prefix = UserDefinedImagePipeline.__name__.upper()
settings = self._generate_fake_settings(prefix=prefix)
settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = UserDefinedImagePipeline.from_crawler(
get_crawler(None, settings)
)
@ -453,27 +447,27 @@ class TestImagesPipelineCustomSettings:
assert custom_value != self.default_pipeline_settings[pipe_attr]
assert getattr(user_pipeline, pipe_attr.lower()) == custom_value
def test_custom_settings_and_class_attrs_for_subclasses(self):
def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path):
"""
If there are custom settings for subclass AND class attributes
setting keys are preferred and override attributes.
"""
pipeline_cls = self._generate_fake_pipeline_subclass()
prefix = pipeline_cls.__name__.upper()
settings = self._generate_fake_settings(prefix=prefix)
settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
for pipe_attr, settings_attr in self.img_cls_attribute_names:
custom_value = settings.get(prefix + "_" + settings_attr)
assert custom_value != self.default_pipeline_settings[pipe_attr]
assert getattr(user_pipeline, pipe_attr.lower()) == custom_value
def test_cls_attrs_with_DEFAULT_prefix(self):
def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path):
class UserDefinedImagePipeline(ImagesPipeline):
DEFAULT_IMAGES_URLS_FIELD = "something"
DEFAULT_IMAGES_RESULT_FIELD = "something_else"
pipeline = UserDefinedImagePipeline.from_crawler(
get_crawler(None, {"IMAGES_STORE": self.tempdir})
get_crawler(None, {"IMAGES_STORE": tmp_path})
)
assert (
pipeline.images_result_field
@ -484,12 +478,12 @@ class TestImagesPipelineCustomSettings:
== UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD
)
def test_user_defined_subclass_default_key_names(self):
def test_user_defined_subclass_default_key_names(self, tmp_path):
"""Test situation when user defines subclass of ImagePipeline,
but uses attribute names for default pipeline (without prefixing
them with pipeline class name).
"""
settings = self._generate_fake_settings()
settings = self._generate_fake_settings(tmp_path)
class UserPipe(ImagesPipeline):
pass

16
tests/test_utils_template.py
View File

@ -1,24 +1,14 @@
from pathlib import Path
from shutil import rmtree
from tempfile import mkdtemp
from scrapy.utils.template import render_templatefile
class TestUtilsRenderTemplateFile:
def setup_method(self):
self.tmp_path = mkdtemp()
def teardown_method(self):
rmtree(self.tmp_path)
def test_simple_render(self):
def test_simple_render(self, tmp_path):
context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}
template = "from ${project_name}.spiders.${name} import ${classname}"
rendered = "from proj.spiders.spi import TheSpider"
template_path = Path(self.tmp_path, "templ.py.tmpl")
render_path = Path(self.tmp_path, "templ.py")
template_path = tmp_path / "templ.py.tmpl"
render_path = tmp_path / "templ.py"
template_path.write_text(template, encoding="utf8")
assert template_path.is_file() # Failure of test itself