From b6f77806b0ec414a28cfcbac3fa2d928040548d6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 9 Apr 2021 12:19:30 -0300 Subject: [PATCH 1/2] Engine tests: fix item class spider, add minimal type hints --- setup.cfg | 3 --- tests/test_engine.py | 24 +++++++++++------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/setup.cfg b/setup.cfg index 89b4ec57e..1fab6fe22 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,9 +43,6 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True -[mypy-tests.test_engine] -ignore_errors = True - [mypy-tests.test_exporters] ignore_errors = True diff --git a/tests/test_engine.py b/tests/test_engine.py index ef1204f94..c406d2577 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -56,7 +56,7 @@ class TestSpider(Spider): name_re = re.compile(r"

(.*?)

", re.M) price_re = re.compile(r">Price: \$(.*?)<", re.M) - item_cls = TestItem + item_cls: type = TestItem def parse(self, response): xlink = LinkExtractor() @@ -66,15 +66,15 @@ class TestSpider(Spider): yield Request(url=link.url, callback=self.parse_item) def parse_item(self, response): - item = self.item_cls() + adapter = ItemAdapter(self.item_cls()) m = self.name_re.search(response.text) if m: - item['name'] = m.group(1) - item['url'] = response.url + adapter['name'] = m.group(1) + adapter['url'] = response.url m = self.price_re.search(response.text) if m: - item['price'] = m.group(1) - return item + adapter['price'] = m.group(1) + return adapter.item class TestDupeFilterSpider(TestSpider): @@ -87,7 +87,7 @@ class DictItemsSpider(TestSpider): class AttrsItemsSpider(TestSpider): - item_class = AttrsItem + item_cls = AttrsItem try: @@ -97,14 +97,12 @@ except ImportError: else: TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)]) - class DataClassItemsSpider(DictItemsSpider): + class _dataclass_spider(DictItemsSpider): def parse_item(self, response): item = super().parse_item(response) - return TestDataClass( - name=item.get('name'), - url=item.get('url'), - price=item.get('price'), - ) + return TestDataClass(**item) + + DataClassItemsSpider = _dataclass_spider class ItemZeroDivisionErrorSpider(TestSpider): From d8d1dc5b508832598591ae9d3d3694f77f70492d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 12 Apr 2021 10:43:02 -0300 Subject: [PATCH 2/2] Ignore typing warning in test --- tests/test_engine.py | 4 +--- tox.ini | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index c406d2577..b2d1d83c7 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -97,13 +97,11 @@ except ImportError: else: TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)]) - class _dataclass_spider(DictItemsSpider): + class DataClassItemsSpider(DictItemsSpider): # type: ignore[no-redef] def parse_item(self, response): item = super().parse_item(response) return TestDataClass(**item) - DataClassItemsSpider = _dataclass_spider - class ItemZeroDivisionErrorSpider(TestSpider): custom_settings = { diff --git a/tox.ini b/tox.ini index 353977519..5b0606f8f 100644 --- a/tox.ini +++ b/tox.ini @@ -37,7 +37,7 @@ basepython = python3 deps = mypy==0.780 commands = - mypy {posargs: scrapy tests} + mypy --show-error-codes {posargs: scrapy tests} [testenv:security] basepython = python3