mirror of https://github.com/scrapy/scrapy.git
Do not treat percent-encoded characters in feed URIs as format specifiers
Feed URIs were formatted with uri % uri_params, so a percent-encoded character in the URI - e.g. an FTP password containing %23 - was parsed as a printf-style format specifier, crashing FeedExporter.open_spider() with 'TypeError: %u format: a real number is required, not dict', or in some cases (%25s) silently corrupting the URI. Substitute only named placeholders such as %(name)s (including conversion specs like %(batch_id)05d) and the %% escape, leaving any other percent sequence untouched. Both the open_spider() and the batch-delivery call sites are covered. Fixes #5794
This commit is contained in:
parent
870803b7fb
commit
65219c311c
|
|
@ -130,6 +130,10 @@ Any other named parameter gets replaced by the spider attribute of the same
|
|||
name. For example, ``%(site_id)s`` would get replaced by the ``spider.site_id``
|
||||
attribute the moment the feed is being created.
|
||||
|
||||
Only named parameters like these (and the ``%%`` escape) are replaced. Any
|
||||
other ``%`` character, e.g. in a percent-encoded part of the URI such as an
|
||||
FTP password containing ``%23``, is left untouched.
|
||||
|
||||
Here are some examples to illustrate:
|
||||
|
||||
- Store in FTP using one directory per spider:
|
||||
|
|
|
|||
|
|
@ -51,6 +51,25 @@ UriParamsCallableT: TypeAlias = Callable[
|
|||
[dict[str, Any], Spider], dict[str, Any] | None
|
||||
]
|
||||
|
||||
_URI_TEMPLATE_PLACEHOLDER = re.compile(
|
||||
r"%(?:%|\(\w+\)[-#0 +]*(?:\d+)?(?:\.\d+)?[diouxXeEfFgGcrsa])"
|
||||
)
|
||||
|
||||
|
||||
def _format_uri_template(uri_template: str, params: dict[str, Any]) -> str:
|
||||
"""Substitute named printf-style placeholders (e.g. ``%(time)s``) and
|
||||
``%%`` in a feed URI template.
|
||||
|
||||
Unlike plain ``uri_template % params``, any other percent sequence, such
|
||||
as a percent-encoded character in a URI (e.g. ``%23``), is left untouched
|
||||
instead of being interpreted as a format specifier (#5794).
|
||||
"""
|
||||
|
||||
def substitute(match: re.Match[str]) -> str:
|
||||
return match.group(0) % params
|
||||
|
||||
return _URI_TEMPLATE_PLACEHOLDER.sub(substitute, uri_template)
|
||||
|
||||
|
||||
class ItemFilter:
|
||||
"""
|
||||
|
|
@ -514,7 +533,7 @@ class FeedExporter:
|
|||
self.slots.append(
|
||||
self._start_new_batch(
|
||||
batch_id=1,
|
||||
uri=uri % uri_params,
|
||||
uri=_format_uri_template(uri, uri_params),
|
||||
feed_options=feed_options,
|
||||
spider=spider,
|
||||
uri_template=uri,
|
||||
|
|
@ -639,7 +658,7 @@ class FeedExporter:
|
|||
slots.append(
|
||||
self._start_new_batch(
|
||||
batch_id=slot.batch_id + 1,
|
||||
uri=slot.uri_template % uri_params,
|
||||
uri=_format_uri_template(slot.uri_template, uri_params),
|
||||
feed_options=self.feeds[slot.uri_template],
|
||||
spider=spider,
|
||||
uri_template=slot.uri_template,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import pytest
|
|||
|
||||
import scrapy
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.extensions.feedexport import FeedExporter
|
||||
from scrapy.extensions.feedexport import FeedExporter, _format_uri_template
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
|
|
@ -112,6 +112,70 @@ class TestURIParams(ABC):
|
|||
|
||||
assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}"
|
||||
|
||||
def test_percent_encoded_characters(self):
|
||||
"""Percent-encoded characters in the URI (e.g. in an FTP password)
|
||||
must not be interpreted as printf-style format specifiers (#5794)."""
|
||||
settings = self.build_settings(
|
||||
uri="ftp://user:2%23um25%21M%23JZ@ftp.example.com/%(name)s.csv",
|
||||
)
|
||||
crawler, feed_exporter = self._crawler_feed_exporter(settings)
|
||||
spider = scrapy.Spider(self.spider_name)
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", ScrapyDeprecationWarning)
|
||||
feed_exporter.open_spider(spider)
|
||||
|
||||
assert (
|
||||
feed_exporter.slots[0].uri
|
||||
== f"ftp://user:2%23um25%21M%23JZ@ftp.example.com/{self.spider_name}.csv"
|
||||
)
|
||||
|
||||
|
||||
class TestFormatURITemplate:
|
||||
params = {
|
||||
"time": "2026-07-02T00-00-00",
|
||||
"batch_id": 3,
|
||||
"name": "myspider",
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("template", "expected"),
|
||||
[
|
||||
# percent-encoded characters are left untouched (#5794)
|
||||
(
|
||||
"ftp://user:2%23um25%21M%23JZ@ftp.example.com/file.csv",
|
||||
"ftp://user:2%23um25%21M%23JZ@ftp.example.com/file.csv",
|
||||
),
|
||||
# named placeholders are substituted as before
|
||||
(
|
||||
"s3://bucket/%(name)s/%(time)s.json",
|
||||
"s3://bucket/myspider/2026-07-02T00-00-00.json",
|
||||
),
|
||||
# conversion specs of named placeholders keep working
|
||||
(
|
||||
"file:///tmp/batch-%(batch_id)05d.csv",
|
||||
"file:///tmp/batch-00003.csv",
|
||||
),
|
||||
# the %% printf escape keeps working
|
||||
(
|
||||
"file:///tmp/100%%-%(name)s.csv",
|
||||
"file:///tmp/100%-myspider.csv",
|
||||
),
|
||||
# %25s is no longer read as a width-25 %s specifier, which used
|
||||
# to silently inject the whole params dict into the URI
|
||||
(
|
||||
"ftp://user:pa%25ss@host/file-%(name)s.csv",
|
||||
"ftp://user:pa%25ss@host/file-myspider.csv",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_substitution(self, template, expected):
|
||||
assert _format_uri_template(template, self.params) == expected
|
||||
|
||||
def test_missing_key_raises(self):
|
||||
with pytest.raises(KeyError):
|
||||
_format_uri_template("file:///tmp/%(missing)s.csv", self.params)
|
||||
|
||||
|
||||
class TestURIParamsSetting(TestURIParams):
|
||||
deprecated_options = True
|
||||
|
|
|
|||
Loading…
Reference in New Issue