mirror of https://github.com/scrapy/scrapy.git
Adding support for Windows of absolute pathlib.Path objects in FeedExporter (#5939)
This commit is contained in:
parent
1b78e48944
commit
04ee3303e4
|
|
@ -156,8 +156,8 @@ The feeds are stored in the local filesystem.
|
|||
- Required external libraries: none
|
||||
|
||||
Note that for the local filesystem storage (only) you can omit the scheme if
|
||||
you specify an absolute path like ``/tmp/export.csv``. This only works on Unix
|
||||
systems though.
|
||||
you specify an absolute path like ``/tmp/export.csv`` (Unix systems only).
|
||||
Alternatively you can also use a :class:`pathlib.Path` object.
|
||||
|
||||
.. _topics-feed-storage-ftp:
|
||||
|
||||
|
|
|
|||
|
|
@ -382,7 +382,9 @@ class FeedExporter:
|
|||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
uri = str(self.settings["FEED_URI"]) # handle pathlib.Path objects
|
||||
uri = self.settings["FEED_URI"]
|
||||
# handle pathlib.Path objects
|
||||
uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
|
||||
feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")}
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(
|
||||
feed_options, self.settings
|
||||
|
|
@ -392,7 +394,8 @@ class FeedExporter:
|
|||
|
||||
# 'FEEDS' setting takes precedence over 'FEED_URI'
|
||||
for uri, feed_options in self.settings.getdict("FEEDS").items():
|
||||
uri = str(uri) # handle pathlib.Path objects
|
||||
# handle pathlib.Path objects
|
||||
uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(
|
||||
feed_options, self.settings
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2758,6 +2758,31 @@ class FeedExportInitTest(unittest.TestCase):
|
|||
with self.assertRaises(NotConfigured):
|
||||
FeedExporter.from_crawler(crawler)
|
||||
|
||||
def test_absolute_pathlib_as_uri(self):
|
||||
with tempfile.NamedTemporaryFile(suffix="json") as tmp:
|
||||
settings = {
|
||||
"FEEDS": {
|
||||
Path(tmp.name).resolve(): {
|
||||
"format": "json",
|
||||
},
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
exporter = FeedExporter.from_crawler(crawler)
|
||||
self.assertIsInstance(exporter, FeedExporter)
|
||||
|
||||
def test_relative_pathlib_as_uri(self):
|
||||
settings = {
|
||||
"FEEDS": {
|
||||
Path("./items.json"): {
|
||||
"format": "json",
|
||||
},
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
exporter = FeedExporter.from_crawler(crawler)
|
||||
self.assertIsInstance(exporter, FeedExporter)
|
||||
|
||||
|
||||
class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage):
|
||||
def __init__(self, uri):
|
||||
|
|
|
|||
Loading…
Reference in New Issue