Solve the feed Path issue

This commit is contained in:
Adrian Chaves 2026-06-26 09:09:47 +02:00
parent d8d7de2339
commit b13bc58802
2 changed files with 59 additions and 4 deletions

View File

@ -471,7 +471,7 @@ class FeedExporter:
)
uri = self.settings["FEED_URI"]
# handle pathlib.Path objects
uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
uri = str(uri.absolute()) if isinstance(uri, Path) else str(uri)
feed_options = {"format": self.settings["FEED_FORMAT"]}
self.feeds[uri] = feed_complete_default_values_from_settings(
feed_options, self.settings
@ -483,9 +483,9 @@ class FeedExporter:
for settings_uri, feed_options in self.settings.getdict("FEEDS").items():
# handle pathlib.Path objects
uri = (
str(settings_uri)
if not isinstance(settings_uri, Path)
else settings_uri.absolute().as_uri()
str(settings_uri.absolute())
if isinstance(settings_uri, Path)
else str(settings_uri)
)
self.feeds[uri] = feed_complete_default_values_from_settings(
feed_options, self.settings

View File

@ -520,6 +520,61 @@ class TestFeedExport(TestFeedExportBase):
header = self.MyItem.fields.keys()
await self.assertExported(items, header, rows)
@coroutine_test
async def test_pathlib_uri_with_placeholders(self):
feed_dir = Path(self.temp_dir, "pathlib_placeholders")
feed_dir.mkdir()
items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
class TestSpider(scrapy.Spider):
name = "testspider"
def parse(self, response):
yield from items
TestSpider.start_urls = [self.mockserver.url("/")]
settings = {
"FEEDS": {
feed_dir / "%(time)s.json": {"format": "json"},
},
}
crawler = get_crawler(TestSpider, settings)
await crawler.crawl_async()
files = list(feed_dir.iterdir())
assert len(files) == 1
assert "%(time)s" not in files[0].name
assert files[0].suffix == ".json"
@coroutine_test
async def test_pathlib_uri_with_spaces_and_unicode(self):
# A pathlib.Path key with spaces and non-ASCII characters must be kept
# verbatim (not percent-encoded), while %()s placeholders are still
# substituted. %(name)s resolves to the spider name deterministically,
# so the resulting file name can be asserted exactly.
feed_dir = Path(self.temp_dir, "pathlib_spaces_unicode")
feed_dir.mkdir()
items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
class TestSpider(scrapy.Spider):
name = "testspider"
def parse(self, response):
yield from items
TestSpider.start_urls = [self.mockserver.url("/")]
settings = {
"FEEDS": {
feed_dir / "out %(name)s ünïcode.json": {"format": "json"},
},
}
crawler = get_crawler(TestSpider, settings)
await crawler.crawl_async()
files = list(feed_dir.iterdir())
assert len(files) == 1
assert files[0].name == "out testspider ünïcode.json"
@coroutine_test
async def test_export_no_items_not_store_empty(self):
for fmt in ("json", "jsonlines", "xml", "csv"):