diff --git a/pyproject.toml b/pyproject.toml index 82d8056f6..84bf41a94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -374,10 +374,6 @@ ignore = [ "B904", # Use capitalized environment variable "SIM112", - - # Temporarily silenced PT rules - # Use a regular `assert` instead of unittest-style `assertEqual` - "PT009", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index db1f5c419..b69f43438 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -156,7 +156,7 @@ def assert_samelines( category=ScrapyDeprecationWarning, stacklevel=2, ) - testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) + testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) # noqa: PT009 def get_from_asyncio_queue(value: _T) -> Awaitable[_T]: diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 38f0333bb..ad335f852 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,3 +1,4 @@ +from typing import Any from unittest import mock import pytest @@ -171,7 +172,11 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self.crawler) middleware._logerror = mock.MagicMock(side_effect=middleware._logerror) deferred = middleware.process_request(Request("http://site.local"), None) - deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called)) + + def check_called(_: Any) -> None: + assert middleware._logerror.called + + deferred.addCallback(check_called) return deferred def test_robotstxt_immediate_error(self): @@ -202,7 +207,11 @@ Disallow: /some/randome/page.html mw_module_logger.error = mock.MagicMock() d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called)) + + def check_not_called(_: Any) -> None: + assert not mw_module_logger.error.called # type: ignore[attr-defined] + + d.addCallback(check_not_called) return d def test_robotstxt_user_agent_setting(self): diff --git a/tests/test_engine.py b/tests/test_engine.py index 4bac8d273..ba4c6dc40 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -433,10 +433,12 @@ class TestEngine(TestEngineBase): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.open_spider(MySpider(), []) e.start() + + def cb(exc: BaseException) -> None: + assert str(exc), "Engine already running" + try: - yield self.assertFailure(e.start(), RuntimeError).addBoth( - lambda exc: self.assertEqual(str(exc), "Engine already running") - ) + yield self.assertFailure(e.start(), RuntimeError).addBoth(cb) finally: yield e.stop() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e515c16a0..9a582e4b7 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -266,17 +266,11 @@ class TestFilesPipeline(unittest.TestCase): class FilesPipelineTestCaseFieldsMixin: - def setup_method(self): - self.tempdir = mkdtemp() - - def teardown_method(self): - rmtree(self.tempdir) - - def test_item_fields_default(self): + def test_item_fields_default(self, tmp_path): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", file_urls=[url]) pipeline = FilesPipeline.from_crawler( - get_crawler(None, {"FILES_STORE": self.tempdir}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) requests = list(pipeline.get_media_requests(item, None)) assert requests[0].url == url @@ -286,14 +280,14 @@ class FilesPipelineTestCaseFieldsMixin: assert files == [results[0][1]] assert isinstance(item, self.item_class) - def test_item_fields_override_settings(self): + def test_item_fields_override_settings(self, tmp_path): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", custom_file_urls=[url]) pipeline = FilesPipeline.from_crawler( get_crawler( None, { - "FILES_STORE": self.tempdir, + "FILES_STORE": tmp_path, "FILES_URLS_FIELD": "custom_file_urls", "FILES_RESULT_FIELD": "custom_files", }, @@ -368,13 +362,7 @@ class TestFilesPipelineCustomSettings: ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"), } - def setup_method(self): - self.tempdir = mkdtemp() - - def teardown_method(self): - rmtree(self.tempdir) - - def _generate_fake_settings(self, prefix=None): + def _generate_fake_settings(self, tmp_path, prefix=None): def random_string(): return "".join([chr(random.randint(97, 123)) for _ in range(10)]) @@ -382,7 +370,7 @@ class TestFilesPipelineCustomSettings: "FILES_EXPIRES": random.randint(100, 1000), "FILES_URLS_FIELD": random_string(), "FILES_RESULT_FIELD": random_string(), - "FILES_STORE": self.tempdir, + "FILES_STORE": tmp_path, } if not prefix: return settings @@ -400,16 +388,16 @@ class TestFilesPipelineCustomSettings: return UserDefinedFilePipeline - def test_different_settings_for_different_instances(self): + def test_different_settings_for_different_instances(self, tmp_path): """ If there are different instances with different settings they should keep different settings. """ - custom_settings = self._generate_fake_settings() + custom_settings = self._generate_fake_settings(tmp_path) another_pipeline = FilesPipeline.from_crawler( get_crawler(None, custom_settings) ) - one_pipeline = FilesPipeline(self.tempdir, crawler=get_crawler(None)) + one_pipeline = FilesPipeline(tmp_path, crawler=get_crawler(None)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: default_value = self.default_cls_settings[pipe_attr] assert getattr(one_pipeline, pipe_attr) == default_value @@ -417,24 +405,24 @@ class TestFilesPipelineCustomSettings: assert default_value != custom_value assert getattr(another_pipeline, pipe_ins_attr) == custom_value - def test_subclass_attributes_preserved_if_no_settings(self): + def test_subclass_attributes_preserved_if_no_settings(self, tmp_path): """ If subclasses override class attributes and there are no special settings those values should be kept. """ pipe_cls = self._generate_fake_pipeline() - pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": self.tempdir})) + pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": tmp_path})) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: custom_value = getattr(pipe, pipe_ins_attr) assert custom_value != self.default_cls_settings[pipe_attr] assert getattr(pipe, pipe_ins_attr) == getattr(pipe, pipe_attr) - def test_subclass_attrs_preserved_custom_settings(self): + def test_subclass_attrs_preserved_custom_settings(self, tmp_path): """ If file settings are defined but they are not defined for subclass settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline() - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: value = getattr(pipeline, pipe_ins_attr) @@ -442,7 +430,7 @@ class TestFilesPipelineCustomSettings: assert value != self.default_cls_settings[pipe_attr] assert value == setting_value - def test_no_custom_settings_for_subclasses(self): + def test_no_custom_settings_for_subclasses(self, tmp_path): """ If there are no settings for subclass and no subclass attributes, pipeline should use attributes of base class. @@ -452,14 +440,14 @@ class TestFilesPipelineCustomSettings: pass user_pipeline = UserDefinedFilesPipeline.from_crawler( - get_crawler(None, {"FILES_STORE": self.tempdir}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_cls_settings.get(pipe_attr.upper()) assert getattr(user_pipeline, pipe_ins_attr) == custom_value - def test_custom_settings_for_subclasses(self): + def test_custom_settings_for_subclasses(self, tmp_path): """ If there are custom settings for subclass and NO class attributes, pipeline should use custom settings. @@ -469,7 +457,7 @@ class TestFilesPipelineCustomSettings: pass prefix = UserDefinedFilesPipeline.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = UserDefinedFilesPipeline.from_crawler( get_crawler(None, settings) ) @@ -479,14 +467,14 @@ class TestFilesPipelineCustomSettings: assert custom_value != self.default_cls_settings[pipe_attr] assert getattr(user_pipeline, pipe_inst_attr) == custom_value - def test_custom_settings_and_class_attrs_for_subclasses(self): + def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path): """ If there are custom settings for subclass AND class attributes setting keys are preferred and override attributes. """ pipeline_cls = self._generate_fake_pipeline() prefix = pipeline_cls.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for ( pipe_cls_attr, @@ -497,13 +485,13 @@ class TestFilesPipelineCustomSettings: assert custom_value != self.default_cls_settings[pipe_cls_attr] assert getattr(user_pipeline, pipe_inst_attr) == custom_value - def test_cls_attrs_with_DEFAULT_prefix(self): + def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path): class UserDefinedFilesPipeline(FilesPipeline): DEFAULT_FILES_RESULT_FIELD = "this" DEFAULT_FILES_URLS_FIELD = "that" pipeline = UserDefinedFilesPipeline.from_crawler( - get_crawler(None, {"FILES_STORE": self.tempdir}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) assert ( pipeline.files_result_field @@ -514,12 +502,12 @@ class TestFilesPipelineCustomSettings: == UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD ) - def test_user_defined_subclass_default_key_names(self): + def test_user_defined_subclass_default_key_names(self, tmp_path): """Test situation when user defines subclass of FilesPipeline, but uses attribute names for default pipeline (without prefixing them with pipeline class name). """ - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) class UserPipe(FilesPipeline): pass diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index fef6bbbe9..f2ee18bd9 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -314,13 +314,7 @@ class TestImagesPipelineCustomSettings: "IMAGES_RESULT_FIELD": "images", } - def setup_method(self): - self.tempdir = mkdtemp() - - def teardown_method(self): - rmtree(self.tempdir) - - def _generate_fake_settings(self, prefix=None): + def _generate_fake_settings(self, tmp_path, prefix=None): """ :param prefix: string for setting keys :return: dictionary of image pipeline settings @@ -331,7 +325,7 @@ class TestImagesPipelineCustomSettings: settings = { "IMAGES_EXPIRES": random.randint(100, 1000), - "IMAGES_STORE": self.tempdir, + "IMAGES_STORE": tmp_path, "IMAGES_RESULT_FIELD": random_string(), "IMAGES_URLS_FIELD": random_string(), "IMAGES_MIN_WIDTH": random.randint(1, 1000), @@ -368,13 +362,13 @@ class TestImagesPipelineCustomSettings: return UserDefinedImagePipeline - def test_different_settings_for_different_instances(self): + def test_different_settings_for_different_instances(self, tmp_path): """ If there are two instances of ImagesPipeline class with different settings, they should have different settings. """ - custom_settings = self._generate_fake_settings() - default_sts_pipe = ImagesPipeline(self.tempdir, crawler=get_crawler(None)) + custom_settings = self._generate_fake_settings(tmp_path) + default_sts_pipe = ImagesPipeline(tmp_path, crawler=get_crawler(None)) user_sts_pipe = ImagesPipeline.from_crawler(get_crawler(None, custom_settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_default_value = self.default_pipeline_settings.get(pipe_attr) @@ -385,14 +379,14 @@ class TestImagesPipelineCustomSettings: ) assert getattr(user_sts_pipe, pipe_attr.lower()) == custom_value - def test_subclass_attrs_preserved_default_settings(self): + def test_subclass_attrs_preserved_default_settings(self, tmp_path): """ If image settings are not defined at all subclass of ImagePipeline takes values from class attributes. """ pipeline_cls = self._generate_fake_pipeline_subclass() pipeline = pipeline_cls.from_crawler( - get_crawler(None, {"IMAGES_STORE": self.tempdir}) + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to class attribute (uppercase). @@ -400,13 +394,13 @@ class TestImagesPipelineCustomSettings: assert attr_value != self.default_pipeline_settings[pipe_attr] assert attr_value == getattr(pipeline, pipe_attr) - def test_subclass_attrs_preserved_custom_settings(self): + def test_subclass_attrs_preserved_custom_settings(self, tmp_path): """ If image settings are defined but they are not defined for subclass default values taken from settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline_subclass() - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to @@ -416,7 +410,7 @@ class TestImagesPipelineCustomSettings: setings_value = settings.get(settings_attr) assert value == setings_value - def test_no_custom_settings_for_subclasses(self): + def test_no_custom_settings_for_subclasses(self, tmp_path): """ If there are no settings for subclass and no subclass attributes, pipeline should use attributes of base class. @@ -426,14 +420,14 @@ class TestImagesPipelineCustomSettings: pass user_pipeline = UserDefinedImagePipeline.from_crawler( - get_crawler(None, {"IMAGES_STORE": self.tempdir}) + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_pipeline_settings.get(pipe_attr.upper()) assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_custom_settings_for_subclasses(self): + def test_custom_settings_for_subclasses(self, tmp_path): """ If there are custom settings for subclass and NO class attributes, pipeline should use custom settings. @@ -443,7 +437,7 @@ class TestImagesPipelineCustomSettings: pass prefix = UserDefinedImagePipeline.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = UserDefinedImagePipeline.from_crawler( get_crawler(None, settings) ) @@ -453,27 +447,27 @@ class TestImagesPipelineCustomSettings: assert custom_value != self.default_pipeline_settings[pipe_attr] assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_custom_settings_and_class_attrs_for_subclasses(self): + def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path): """ If there are custom settings for subclass AND class attributes setting keys are preferred and override attributes. """ pipeline_cls = self._generate_fake_pipeline_subclass() prefix = pipeline_cls.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: custom_value = settings.get(prefix + "_" + settings_attr) assert custom_value != self.default_pipeline_settings[pipe_attr] assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_cls_attrs_with_DEFAULT_prefix(self): + def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path): class UserDefinedImagePipeline(ImagesPipeline): DEFAULT_IMAGES_URLS_FIELD = "something" DEFAULT_IMAGES_RESULT_FIELD = "something_else" pipeline = UserDefinedImagePipeline.from_crawler( - get_crawler(None, {"IMAGES_STORE": self.tempdir}) + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) assert ( pipeline.images_result_field @@ -484,12 +478,12 @@ class TestImagesPipelineCustomSettings: == UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD ) - def test_user_defined_subclass_default_key_names(self): + def test_user_defined_subclass_default_key_names(self, tmp_path): """Test situation when user defines subclass of ImagePipeline, but uses attribute names for default pipeline (without prefixing them with pipeline class name). """ - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) class UserPipe(ImagesPipeline): pass diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 0b845fdb0..41d9b8933 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,24 +1,14 @@ -from pathlib import Path -from shutil import rmtree -from tempfile import mkdtemp - from scrapy.utils.template import render_templatefile class TestUtilsRenderTemplateFile: - def setup_method(self): - self.tmp_path = mkdtemp() - - def teardown_method(self): - rmtree(self.tmp_path) - - def test_simple_render(self): + def test_simple_render(self, tmp_path): context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"} template = "from ${project_name}.spiders.${name} import ${classname}" rendered = "from proj.spiders.spi import TheSpider" - template_path = Path(self.tmp_path, "templ.py.tmpl") - render_path = Path(self.tmp_path, "templ.py") + template_path = tmp_path / "templ.py.tmpl" + render_path = tmp_path / "templ.py" template_path.write_text(template, encoding="utf8") assert template_path.is_file() # Failure of test itself