fix: -O/-o option does not work with absolute paths on Windows (#5971)

This commit is contained in:
Laerte Pereira 2023-07-24 04:53:41 -03:00 committed by GitHub
parent af2aa4b421
commit 7522aeed35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 63 additions and 4 deletions

View File

@ -9,7 +9,7 @@ import re
import sys
import warnings
from datetime import datetime
from pathlib import Path
from pathlib import Path, PureWindowsPath
from tempfile import NamedTemporaryFile
from typing import IO, Any, Callable, List, Optional, Tuple, Union
from urllib.parse import unquote, urlparse
@ -615,7 +615,7 @@ class FeedExporter:
def _storage_supported(self, uri, feed_options):
scheme = urlparse(uri).scheme
if scheme in self.storages:
if scheme in self.storages or PureWindowsPath(uri).drive:
try:
self._get_storage(uri, feed_options)
return True
@ -633,7 +633,7 @@ class FeedExporter:
It supports not passing the *feed_options* parameters to classes that
do not support it, and issuing a deprecation warning instead.
"""
feedcls = self.storages[urlparse(uri).scheme]
feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"])
crawler = getattr(self, "crawler", None)
def build_instance(builder, *preargs):

View File

@ -202,7 +202,8 @@ def feed_process_params_from_cli(
for element in output:
try:
feed_uri, feed_format = element.rsplit(":", 1)
except ValueError:
check_valid_format(feed_format)
except (ValueError, UsageError):
feed_uri = element
feed_format = Path(element).suffix.replace(".", "")
else:

View File

@ -907,6 +907,64 @@ class MySpider(scrapy.Spider):
log = self.get_log(spider_code, args=args)
self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log)
@skipIf(platform.system() == "Windows", reason="Linux only")
def test_absolute_path_linux(self):
spider_code = """
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ["data:,"]
def parse(self, response):
yield {"hello": "world"}
"""
temp_dir = mkdtemp()
args = ["-o", f"{temp_dir}/output1.json:json"]
log = self.get_log(spider_code, args=args)
self.assertIn(
f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json",
log,
)
args = ["-o", f"{temp_dir}/output2.json"]
log = self.get_log(spider_code, args=args)
self.assertIn(
f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json",
log,
)
@skipIf(platform.system() != "Windows", reason="Windows only")
def test_absolute_path_windows(self):
spider_code = """
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ["data:,"]
def parse(self, response):
yield {"hello": "world"}
"""
temp_dir = mkdtemp()
args = ["-o", f"{temp_dir}\\output1.json:json"]
log = self.get_log(spider_code, args=args)
self.assertIn(
f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json",
log,
)
args = ["-o", f"{temp_dir}\\output2.json"]
log = self.get_log(spider_code, args=args)
self.assertIn(
f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json",
log,
)
@skipIf(platform.system() != "Windows", "Windows required for .pyw files")
class WindowsRunSpiderCommandTest(RunSpiderCommandTest):