mirror of https://github.com/scrapy/scrapy.git
fix: -O/-o option does not work with absolute paths on Windows (#5971)
This commit is contained in:
parent
af2aa4b421
commit
7522aeed35
|
|
@ -9,7 +9,7 @@ import re
|
|||
import sys
|
||||
import warnings
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PureWindowsPath
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import IO, Any, Callable, List, Optional, Tuple, Union
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
|
@ -615,7 +615,7 @@ class FeedExporter:
|
|||
|
||||
def _storage_supported(self, uri, feed_options):
|
||||
scheme = urlparse(uri).scheme
|
||||
if scheme in self.storages:
|
||||
if scheme in self.storages or PureWindowsPath(uri).drive:
|
||||
try:
|
||||
self._get_storage(uri, feed_options)
|
||||
return True
|
||||
|
|
@ -633,7 +633,7 @@ class FeedExporter:
|
|||
It supports not passing the *feed_options* parameters to classes that
|
||||
do not support it, and issuing a deprecation warning instead.
|
||||
"""
|
||||
feedcls = self.storages[urlparse(uri).scheme]
|
||||
feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"])
|
||||
crawler = getattr(self, "crawler", None)
|
||||
|
||||
def build_instance(builder, *preargs):
|
||||
|
|
|
|||
|
|
@ -202,7 +202,8 @@ def feed_process_params_from_cli(
|
|||
for element in output:
|
||||
try:
|
||||
feed_uri, feed_format = element.rsplit(":", 1)
|
||||
except ValueError:
|
||||
check_valid_format(feed_format)
|
||||
except (ValueError, UsageError):
|
||||
feed_uri = element
|
||||
feed_format = Path(element).suffix.replace(".", "")
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -907,6 +907,64 @@ class MySpider(scrapy.Spider):
|
|||
log = self.get_log(spider_code, args=args)
|
||||
self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log)
|
||||
|
||||
@skipIf(platform.system() == "Windows", reason="Linux only")
|
||||
def test_absolute_path_linux(self):
|
||||
spider_code = """
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = 'myspider'
|
||||
|
||||
start_urls = ["data:,"]
|
||||
|
||||
def parse(self, response):
|
||||
yield {"hello": "world"}
|
||||
"""
|
||||
temp_dir = mkdtemp()
|
||||
|
||||
args = ["-o", f"{temp_dir}/output1.json:json"]
|
||||
log = self.get_log(spider_code, args=args)
|
||||
self.assertIn(
|
||||
f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json",
|
||||
log,
|
||||
)
|
||||
|
||||
args = ["-o", f"{temp_dir}/output2.json"]
|
||||
log = self.get_log(spider_code, args=args)
|
||||
self.assertIn(
|
||||
f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json",
|
||||
log,
|
||||
)
|
||||
|
||||
@skipIf(platform.system() != "Windows", reason="Windows only")
|
||||
def test_absolute_path_windows(self):
|
||||
spider_code = """
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = 'myspider'
|
||||
|
||||
start_urls = ["data:,"]
|
||||
|
||||
def parse(self, response):
|
||||
yield {"hello": "world"}
|
||||
"""
|
||||
temp_dir = mkdtemp()
|
||||
|
||||
args = ["-o", f"{temp_dir}\\output1.json:json"]
|
||||
log = self.get_log(spider_code, args=args)
|
||||
self.assertIn(
|
||||
f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json",
|
||||
log,
|
||||
)
|
||||
|
||||
args = ["-o", f"{temp_dir}\\output2.json"]
|
||||
log = self.get_log(spider_code, args=args)
|
||||
self.assertIn(
|
||||
f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json",
|
||||
log,
|
||||
)
|
||||
|
||||
|
||||
@skipIf(platform.system() != "Windows", "Windows required for .pyw files")
|
||||
class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
|
||||
|
|
|
|||
Loading…
Reference in New Issue