From 7522aeed357d90ed95ee10d3b5a506f1b1049d1f Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Mon, 24 Jul 2023 04:53:41 -0300 Subject: [PATCH] fix: -O/-o option does not work with absolute paths on Windows (#5971) --- scrapy/extensions/feedexport.py | 6 ++-- scrapy/utils/conf.py | 3 +- tests/test_commands.py | 58 +++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 6e391d3dc..c81f396cb 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -9,7 +9,7 @@ import re import sys import warnings from datetime import datetime -from pathlib import Path +from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile from typing import IO, Any, Callable, List, Optional, Tuple, Union from urllib.parse import unquote, urlparse @@ -615,7 +615,7 @@ class FeedExporter: def _storage_supported(self, uri, feed_options): scheme = urlparse(uri).scheme - if scheme in self.storages: + if scheme in self.storages or PureWindowsPath(uri).drive: try: self._get_storage(uri, feed_options) return True @@ -633,7 +633,7 @@ class FeedExporter: It supports not passing the *feed_options* parameters to classes that do not support it, and issuing a deprecation warning instead. """ - feedcls = self.storages[urlparse(uri).scheme] + feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) crawler = getattr(self, "crawler", None) def build_instance(builder, *preargs): diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 3ade1d105..05d43e456 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -202,7 +202,8 @@ def feed_process_params_from_cli( for element in output: try: feed_uri, feed_format = element.rsplit(":", 1) - except ValueError: + check_valid_format(feed_format) + except (ValueError, UsageError): feed_uri = element feed_format = Path(element).suffix.replace(".", "") else: diff --git a/tests/test_commands.py b/tests/test_commands.py index 03d768d1a..b1d7be628 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -907,6 +907,64 @@ class MySpider(scrapy.Spider): log = self.get_log(spider_code, args=args) self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log) + @skipIf(platform.system() == "Windows", reason="Linux only") + def test_absolute_path_linux(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}/output1.json:json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json", + log, + ) + + args = ["-o", f"{temp_dir}/output2.json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json", + log, + ) + + @skipIf(platform.system() != "Windows", reason="Windows only") + def test_absolute_path_windows(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}\\output1.json:json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json", + log, + ) + + args = ["-o", f"{temp_dir}\\output2.json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json", + log, + ) + @skipIf(platform.system() != "Windows", "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest):