From 7e1b2e6a7810cd0dc07fbf737de2c082248a9f61 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Jul 2026 01:00:44 +0500 Subject: [PATCH] Extract MyItem. --- tests/test_feedexport.py | 55 ++++++++++++++-------------------- tests/test_feedexport_batch.py | 21 ++++++------- tests/utils/feedexport.py | 13 ++++++++ 3 files changed, 46 insertions(+), 43 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b9f8c3aa4..853243b97 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -37,7 +37,7 @@ from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import ItemSpider from tests.utils.decorators import coroutine_test, inline_callbacks_test -from tests.utils.feedexport import path_to_url, printf_escape +from tests.utils.feedexport import MyItem, MyItem2, path_to_url, printf_escape if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Iterable @@ -105,15 +105,6 @@ class LogOnStoreFileStorage: class TestFeedExportBase(ABC): mockserver: MockServer - class MyItem(scrapy.Item): - foo = scrapy.Field() - egg = scrapy.Field() - baz = scrapy.Field() - - class MyItem2(scrapy.Item): - foo = scrapy.Field() - hello = scrapy.Field() - def _random_temp_filename(self, inter_dir="") -> Path: chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = "".join(chars) @@ -508,21 +499,21 @@ class TestFeedExport(TestFeedExportBase): async def test_export_items(self): # feed exporters use field names from Item items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), ] rows = [ {"egg": "spam1", "foo": "bar1", "baz": ""}, {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, ] - header = self.MyItem.fields.keys() + header = MyItem.fields.keys() await self.assertExported(items, header, rows) @coroutine_test async def test_pathlib_uri_with_placeholders(self): feed_dir = Path(self.temp_dir, "pathlib_placeholders") feed_dir.mkdir() - items = [self.MyItem({"foo": "bar1", "egg": "spam1"})] + items = [MyItem({"foo": "bar1", "egg": "spam1"})] class TestSpider(scrapy.Spider): name = "testspider" @@ -552,7 +543,7 @@ class TestFeedExport(TestFeedExportBase): # so the resulting file name can be asserted exactly. feed_dir = Path(self.temp_dir, "pathlib_spaces_unicode") feed_dir.mkdir() - items = [self.MyItem({"foo": "bar1", "egg": "spam1"})] + items = [MyItem({"foo": "bar1", "egg": "spam1"})] class TestSpider(scrapy.Spider): name = "testspider" @@ -581,7 +572,7 @@ class TestFeedExport(TestFeedExportBase): # and #5794. feed_dir = Path(self.temp_dir, "dir with spaces") feed_dir.mkdir() - items = [self.MyItem({"foo": "bar1", "egg": "spam1"})] + items = [MyItem({"foo": "bar1", "egg": "spam1"})] class TestSpider(scrapy.Spider): name = "testspider" @@ -618,7 +609,7 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_start_finish_exporting_items(self): items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem({"foo": "bar1", "egg": "spam1"}), ] settings = { "FEEDS": { @@ -656,7 +647,7 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_start_finish_exporting_items_exception(self): items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem({"foo": "bar1", "egg": "spam1"}), ] settings = { "FEEDS": { @@ -734,15 +725,15 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_export_multiple_item_classes(self): items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem2({"hello": "world2", "foo": "bar2"}), - self.MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem2({"hello": "world2", "foo": "bar2"}), + MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}), {"hello": "world4", "egg": "spam4"}, ] # by default, Scrapy uses fields of the first Item for CSV and # all fields for JSON Lines - header = self.MyItem.fields.keys() + header = MyItem.fields.keys() rows_csv = [ {"egg": "spam1", "foo": "bar1", "baz": ""}, {"egg": "", "foo": "bar2", "baz": ""}, @@ -817,8 +808,8 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_export_based_on_item_classes(self): items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem2({"hello": "world2", "foo": "bar2"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem2({"hello": "world2", "foo": "bar2"}), {"hello": "world3", "egg": "spam3"}, ] @@ -840,15 +831,15 @@ class TestFeedExport(TestFeedExportBase): "FEEDS": { self._random_temp_filename(): { "format": "csv", - "item_classes": [self.MyItem], + "item_classes": [MyItem], }, self._random_temp_filename(): { "format": "json", - "item_classes": [self.MyItem2], + "item_classes": [MyItem2], }, self._random_temp_filename(): { "format": "jsonlines", - "item_classes": [self.MyItem, self.MyItem2], + "item_classes": [MyItem, MyItem2], }, self._random_temp_filename(): { "format": "xml", @@ -863,13 +854,11 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_export_based_on_custom_filters(self): items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem2({"hello": "world2", "foo": "bar2"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem2({"hello": "world2", "foo": "bar2"}), {"hello": "world3", "egg": "spam3"}, ] - MyItem = self.MyItem - class CustomFilter1: def __init__(self, feed_options): pass @@ -909,7 +898,7 @@ class TestFeedExport(TestFeedExportBase): }, self._random_temp_filename(): { "format": "jsonlines", - "item_classes": [self.MyItem, self.MyItem2], + "item_classes": [MyItem, MyItem2], "item_filter": CustomFilter3, }, }, @@ -948,7 +937,7 @@ class TestFeedExport(TestFeedExportBase): # FEED_EXPORT_FIELDS option allows to order export fields # and to select a subset of fields to export, both for Items and dicts. - for item_cls in [self.MyItem, dict]: + for item_cls in [MyItem, dict]: items = [ item_cls({"foo": "bar1", "egg": "spam1"}), item_cls({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 469985288..6e0536110 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -24,6 +24,7 @@ from scrapy.utils.test import get_crawler from tests.spiders import ItemSpider from tests.test_feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.feedexport import MyItem if TYPE_CHECKING: from os import PathLike @@ -197,9 +198,9 @@ class TestBatchDeliveries(TestFeedExportBase): async def test_export_items(self): """Test partial deliveries in all supported formats""" items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + MyItem({"foo": "bar3", "baz": "quux3"}), ] rows = [ {"egg": "spam1", "foo": "bar1", "baz": ""}, @@ -207,7 +208,7 @@ class TestBatchDeliveries(TestFeedExportBase): {"foo": "bar3", "baz": "quux3", "egg": ""}, ] settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2} - header = self.MyItem.fields.keys() + header = MyItem.fields.keys() await self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): @@ -349,9 +350,9 @@ class TestBatchDeliveries(TestFeedExportBase): So %(batch_id)d replaced with the current id. """ items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + MyItem({"foo": "bar3", "baz": "quux3"}), ] settings = { "FEEDS": { @@ -387,9 +388,9 @@ class TestBatchDeliveries(TestFeedExportBase): def test_s3_export(self): bucket = "mybucket" items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + MyItem({"foo": "bar3", "baz": "quux3"}), ] class CustomS3FeedStorage(S3FeedStorage): diff --git a/tests/utils/feedexport.py b/tests/utils/feedexport.py index 79f9208f5..515082217 100644 --- a/tests/utils/feedexport.py +++ b/tests/utils/feedexport.py @@ -4,6 +4,8 @@ from typing import TYPE_CHECKING from urllib.parse import urljoin from urllib.request import pathname2url +from scrapy import Field, Item + if TYPE_CHECKING: from pathlib import Path @@ -14,3 +16,14 @@ def path_to_url(path: str | Path) -> str: def printf_escape(s: str) -> str: return s.replace("%", "%%") + + +class MyItem(Item): + foo = Field() + egg = Field() + baz = Field() + + +class MyItem2(Item): + foo = Field() + hello = Field()