From e248360e6e3dbb36fab185caf131707195fa6a26 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 18 Jul 2022 23:49:08 +0500 Subject: [PATCH 1/2] remove compatibility code from tests for the case dataclasses module is not available It was Python 3.6 compat code, and Python 3.6 support is dropped. --- tests/test_exporters.py | 32 ++++++++++++++--------------- tests/test_loader.py | 23 +++++++-------------- tests/test_pipeline_files.py | 37 ++++++++++++---------------------- tests/test_pipeline_images.py | 38 ++++++++++++----------------------- tests/test_utils_serialize.py | 18 +++++++---------- 5 files changed, 55 insertions(+), 93 deletions(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 096cd3116..69ac928c3 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -4,6 +4,7 @@ import marshal import pickle import tempfile import unittest +import dataclasses from io import BytesIO from datetime import datetime from warnings import catch_warnings, filterwarnings @@ -21,31 +22,30 @@ from scrapy.exporters import ( ) +def custom_serializer(value): + return str(int(value) + 2) + + class TestItem(Item): name = Field() age = Field() -def custom_serializer(value): - return str(int(value) + 2) - - class CustomFieldItem(Item): name = Field() age = Field(serializer=custom_serializer) -try: - from dataclasses import make_dataclass, field -except ImportError: - TestDataClass = None - CustomFieldDataclass = None -else: - TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)]) - CustomFieldDataclass = make_dataclass( - "CustomFieldDataclass", - [("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))] - ) +@dataclasses.dataclass +class TestDataClass: + name: str + age: int + + +@dataclasses.dataclass +class CustomFieldDataclass: + name: str + age: int = dataclasses.field(metadata={"serializer": custom_serializer}) class BaseItemExporterTest(unittest.TestCase): @@ -54,8 +54,6 @@ class BaseItemExporterTest(unittest.TestCase): custom_field_item_class = CustomFieldItem def setUp(self): - if self.item_class is None: - raise unittest.SkipTest("item class is None") self.i = self.item_class(name='John\xa3', age='22') self.output = BytesIO() self.ie = self._get_exporter() diff --git a/tests/test_loader.py b/tests/test_loader.py index f7ab1f236..c0937b349 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,4 +1,5 @@ import unittest +import dataclasses import attr from itemadapter import ItemAdapter @@ -10,13 +11,6 @@ from scrapy.loader import ItemLoader from scrapy.selector import Selector -try: - from dataclasses import make_dataclass, field as dataclass_field -except ImportError: - make_dataclass = None - dataclass_field = None - - # test items class NameItem(Item): name = Field() @@ -41,6 +35,11 @@ class AttrsNameItem: name = attr.ib(default="") +@dataclasses.dataclass +class TestDataClass: + name: list = dataclasses.field(default_factory=list) + + # test item loaders class NameItemLoader(ItemLoader): default_item_class = TestItem @@ -187,16 +186,8 @@ class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase item_class = AttrsNameItem -@unittest.skipIf(not make_dataclass, "dataclasses module is not available") class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if make_dataclass: - self.item_class = make_dataclass( - "TestDataClass", - [("name", list, dataclass_field(default_factory=list))], - ) + item_class = TestDataClass class BaseNoInputReprocessingLoader(ItemLoader): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4228173ed..5d381c018 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -7,6 +7,7 @@ from shutil import rmtree from tempfile import mkdtemp from unittest import mock, skipIf from urllib.parse import urlparse +import dataclasses import attr from itemadapter import ItemAdapter @@ -32,13 +33,6 @@ from scrapy.utils.test import ( ) -try: - from dataclasses import make_dataclass, field as dataclass_field -except ImportError: - make_dataclass = None - dataclass_field = None - - def _mocked_download_func(request, info): response = request.meta.get('response') return response() if callable(response) else response @@ -226,24 +220,19 @@ class FilesPipelineTestCaseFieldsItem(FilesPipelineTestCaseFieldsMixin, unittest item_class = FilesPipelineTestItem -@skipIf(not make_dataclass, "dataclasses module is not available") -class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): +@dataclasses.dataclass +class FilesPipelineTestDataClass: + name: str + # default fields + file_urls: list = dataclasses.field(default_factory=list) + files: list = dataclasses.field(default_factory=list) + # overridden fields + custom_file_urls: list = dataclasses.field(default_factory=list) + custom_files: list = dataclasses.field(default_factory=list) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if make_dataclass: - self.item_class = make_dataclass( - "FilesPipelineTestDataClass", - [ - ("name", str), - # default fields - ("file_urls", list, dataclass_field(default_factory=list)), - ("files", list, dataclass_field(default_factory=list)), - # overridden fields - ("custom_file_urls", list, dataclass_field(default_factory=list)), - ("custom_files", list, dataclass_field(default_factory=list)), - ], - ) + +class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): + item_class = FilesPipelineTestDataClass @attr.s diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index dd94d296b..e6f5bea21 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -4,6 +4,7 @@ import random from shutil import rmtree from tempfile import mkdtemp from unittest import skipIf +import dataclasses import attr from itemadapter import ItemAdapter @@ -16,13 +17,6 @@ from scrapy.settings import Settings from scrapy.utils.python import to_bytes -try: - from dataclasses import make_dataclass, field as dataclass_field -except ImportError: - make_dataclass = None - dataclass_field = None - - try: from PIL import Image except ImportError: @@ -203,25 +197,19 @@ class ImagesPipelineTestCaseFieldsItem(ImagesPipelineTestCaseFieldsMixin, unitte item_class = ImagesPipelineTestItem -@skipIf(not make_dataclass, "dataclasses module is not available") -class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): - item_class = None +@dataclasses.dataclass +class ImagesPipelineTestDataClass: + name: str + # default fields + image_urls: list = dataclasses.field(default_factory=list) + images: list = dataclasses.field(default_factory=list) + # overridden fields + custom_image_urls: list = dataclasses.field(default_factory=list) + custom_images: list = dataclasses.field(default_factory=list) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if make_dataclass: - self.item_class = make_dataclass( - "FilesPipelineTestDataClass", - [ - ("name", str), - # default fields - ("image_urls", list, dataclass_field(default_factory=list)), - ("images", list, dataclass_field(default_factory=list)), - # overridden fields - ("custom_image_urls", list, dataclass_field(default_factory=list)), - ("custom_images", list, dataclass_field(default_factory=list)), - ], - ) + +class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): + item_class = ImagesPipelineTestDataClass @attr.s diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index daf022aee..a51de1877 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -1,6 +1,7 @@ import datetime import json import unittest +import dataclasses from decimal import Decimal import attr @@ -10,12 +11,6 @@ from scrapy.http import Request, Response from scrapy.utils.serialize import ScrapyJSONEncoder -try: - from dataclasses import make_dataclass -except ImportError: - make_dataclass = None - - class JsonEncoderTestCase(unittest.TestCase): def setUp(self): @@ -56,12 +51,13 @@ class JsonEncoderTestCase(unittest.TestCase): self.assertIn(r.url, rs) self.assertIn(str(r.status), rs) - @unittest.skipIf(not make_dataclass, "No dataclass support") def test_encode_dataclass_item(self): - TestDataClass = make_dataclass( - "TestDataClass", - [("name", str), ("url", str), ("price", int)], - ) + @dataclasses.dataclass + class TestDataClass: + name: str + url: str + price: int + item = TestDataClass(name="Product", url="http://product.org", price=1) encoded = self.encoder.encode(item) self.assertEqual( From 105468959363ee50b597038ac30fd32d3ea1b1f2 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 18 Jul 2022 23:53:30 +0500 Subject: [PATCH 2/2] remove unused imports thanks flake8! --- tests/test_pipeline_files.py | 2 +- tests/test_pipeline_images.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 5d381c018..d641e7a43 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -5,7 +5,7 @@ from datetime import datetime from io import BytesIO from shutil import rmtree from tempfile import mkdtemp -from unittest import mock, skipIf +from unittest import mock from urllib.parse import urlparse import dataclasses diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index e6f5bea21..0082e7a4e 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -3,7 +3,6 @@ import io import random from shutil import rmtree from tempfile import mkdtemp -from unittest import skipIf import dataclasses import attr