Do not treat percent-encoded characters in feed URIs as format specifiers

Feed URIs were formatted with uri % uri_params, so a percent-encoded
character in the URI - e.g. an FTP password containing %23 - was parsed
as a printf-style format specifier, crashing FeedExporter.open_spider()
with 'TypeError: %u format: a real number is required, not dict', or in
some cases (%25s) silently corrupting the URI.

Substitute only named placeholders such as %(name)s (including
conversion specs like %(batch_id)05d) and the %% escape, leaving any
other percent sequence untouched. Both the open_spider() and the
batch-delivery call sites are covered.

Fixes #5794
This commit is contained in:
Apoorv Darshan 2026-07-02 18:18:18 +05:30
parent 870803b7fb
commit 65219c311c
No known key found for this signature in database
3 changed files with 90 additions and 3 deletions

View File

@ -130,6 +130,10 @@ Any other named parameter gets replaced by the spider attribute of the same
name. For example, ``%(site_id)s`` would get replaced by the ``spider.site_id``
attribute the moment the feed is being created.
Only named parameters like these (and the ``%%`` escape) are replaced. Any
other ``%`` character, e.g. in a percent-encoded part of the URI such as an
FTP password containing ``%23``, is left untouched.
Here are some examples to illustrate:
- Store in FTP using one directory per spider:

View File

@ -51,6 +51,25 @@ UriParamsCallableT: TypeAlias = Callable[
[dict[str, Any], Spider], dict[str, Any] | None
]
_URI_TEMPLATE_PLACEHOLDER = re.compile(
r"%(?:%|\(\w+\)[-#0 +]*(?:\d+)?(?:\.\d+)?[diouxXeEfFgGcrsa])"
)
def _format_uri_template(uri_template: str, params: dict[str, Any]) -> str:
"""Substitute named printf-style placeholders (e.g. ``%(time)s``) and
``%%`` in a feed URI template.
Unlike plain ``uri_template % params``, any other percent sequence, such
as a percent-encoded character in a URI (e.g. ``%23``), is left untouched
instead of being interpreted as a format specifier (#5794).
"""
def substitute(match: re.Match[str]) -> str:
return match.group(0) % params
return _URI_TEMPLATE_PLACEHOLDER.sub(substitute, uri_template)
class ItemFilter:
"""
@ -514,7 +533,7 @@ class FeedExporter:
self.slots.append(
self._start_new_batch(
batch_id=1,
uri=uri % uri_params,
uri=_format_uri_template(uri, uri_params),
feed_options=feed_options,
spider=spider,
uri_template=uri,
@ -639,7 +658,7 @@ class FeedExporter:
slots.append(
self._start_new_batch(
batch_id=slot.batch_id + 1,
uri=slot.uri_template % uri_params,
uri=_format_uri_template(slot.uri_template, uri_params),
feed_options=self.feeds[slot.uri_template],
spider=spider,
uri_template=slot.uri_template,

View File

@ -7,7 +7,7 @@ import pytest
import scrapy
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.feedexport import FeedExporter
from scrapy.extensions.feedexport import FeedExporter, _format_uri_template
from scrapy.utils.test import get_crawler
@ -112,6 +112,70 @@ class TestURIParams(ABC):
assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}"
def test_percent_encoded_characters(self):
"""Percent-encoded characters in the URI (e.g. in an FTP password)
must not be interpreted as printf-style format specifiers (#5794)."""
settings = self.build_settings(
uri="ftp://user:2%23um25%21M%23JZ@ftp.example.com/%(name)s.csv",
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)
assert (
feed_exporter.slots[0].uri
== f"ftp://user:2%23um25%21M%23JZ@ftp.example.com/{self.spider_name}.csv"
)
class TestFormatURITemplate:
params = {
"time": "2026-07-02T00-00-00",
"batch_id": 3,
"name": "myspider",
}
@pytest.mark.parametrize(
("template", "expected"),
[
# percent-encoded characters are left untouched (#5794)
(
"ftp://user:2%23um25%21M%23JZ@ftp.example.com/file.csv",
"ftp://user:2%23um25%21M%23JZ@ftp.example.com/file.csv",
),
# named placeholders are substituted as before
(
"s3://bucket/%(name)s/%(time)s.json",
"s3://bucket/myspider/2026-07-02T00-00-00.json",
),
# conversion specs of named placeholders keep working
(
"file:///tmp/batch-%(batch_id)05d.csv",
"file:///tmp/batch-00003.csv",
),
# the %% printf escape keeps working
(
"file:///tmp/100%%-%(name)s.csv",
"file:///tmp/100%-myspider.csv",
),
# %25s is no longer read as a width-25 %s specifier, which used
# to silently inject the whole params dict into the URI
(
"ftp://user:pa%25ss@host/file-%(name)s.csv",
"ftp://user:pa%25ss@host/file-myspider.csv",
),
],
)
def test_substitution(self, template, expected):
assert _format_uri_template(template, self.params) == expected
def test_missing_key_raises(self):
with pytest.raises(KeyError):
_format_uri_template("file:///tmp/%(missing)s.csv", self.params)
class TestURIParamsSetting(TestURIParams):
deprecated_options = True