Adding support for Windows of absolute pathlib.Path objects in FeedExporter (#5939)

This commit is contained in:
Alex 2023-06-21 22:04:06 -07:00 committed by GitHub
parent 1b78e48944
commit 04ee3303e4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 32 additions and 4 deletions

View File

@ -156,8 +156,8 @@ The feeds are stored in the local filesystem.
- Required external libraries: none
Note that for the local filesystem storage (only) you can omit the scheme if
you specify an absolute path like ``/tmp/export.csv``. This only works on Unix
systems though.
you specify an absolute path like ``/tmp/export.csv`` (Unix systems only).
Alternatively you can also use a :class:`pathlib.Path` object.
.. _topics-feed-storage-ftp:

View File

@ -382,7 +382,9 @@ class FeedExporter:
category=ScrapyDeprecationWarning,
stacklevel=2,
)
uri = str(self.settings["FEED_URI"]) # handle pathlib.Path objects
uri = self.settings["FEED_URI"]
# handle pathlib.Path objects
uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")}
self.feeds[uri] = feed_complete_default_values_from_settings(
feed_options, self.settings
@ -392,7 +394,8 @@ class FeedExporter:
# 'FEEDS' setting takes precedence over 'FEED_URI'
for uri, feed_options in self.settings.getdict("FEEDS").items():
uri = str(uri) # handle pathlib.Path objects
# handle pathlib.Path objects
uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
self.feeds[uri] = feed_complete_default_values_from_settings(
feed_options, self.settings
)

View File

@ -2758,6 +2758,31 @@ class FeedExportInitTest(unittest.TestCase):
with self.assertRaises(NotConfigured):
FeedExporter.from_crawler(crawler)
def test_absolute_pathlib_as_uri(self):
with tempfile.NamedTemporaryFile(suffix="json") as tmp:
settings = {
"FEEDS": {
Path(tmp.name).resolve(): {
"format": "json",
},
},
}
crawler = get_crawler(settings_dict=settings)
exporter = FeedExporter.from_crawler(crawler)
self.assertIsInstance(exporter, FeedExporter)
def test_relative_pathlib_as_uri(self):
settings = {
"FEEDS": {
Path("./items.json"): {
"format": "json",
},
},
}
crawler = get_crawler(settings_dict=settings)
exporter = FeedExporter.from_crawler(crawler)
self.assertIsInstance(exporter, FeedExporter)
class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage):
def __init__(self, uri):