diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 6ce5255b9..d8f671bc7 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -130,6 +130,10 @@ Any other named parameter gets replaced by the spider attribute of the same name. For example, ``%(site_id)s`` would get replaced by the ``spider.site_id`` attribute the moment the feed is being created. +Only named parameters like these (and the ``%%`` escape) are replaced. Any +other ``%`` character, e.g. in a percent-encoded part of the URI such as an +FTP password containing ``%23``, is left untouched. + Here are some examples to illustrate: - Store in FTP using one directory per spider: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fa708a2e1..1aa5f5291 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -51,6 +51,25 @@ UriParamsCallableT: TypeAlias = Callable[ [dict[str, Any], Spider], dict[str, Any] | None ] +_URI_TEMPLATE_PLACEHOLDER = re.compile( + r"%(?:%|\(\w+\)[-#0 +]*(?:\d+)?(?:\.\d+)?[diouxXeEfFgGcrsa])" +) + + +def _format_uri_template(uri_template: str, params: dict[str, Any]) -> str: + """Substitute named printf-style placeholders (e.g. ``%(time)s``) and + ``%%`` in a feed URI template. + + Unlike plain ``uri_template % params``, any other percent sequence, such + as a percent-encoded character in a URI (e.g. ``%23``), is left untouched + instead of being interpreted as a format specifier (#5794). + """ + + def substitute(match: re.Match[str]) -> str: + return match.group(0) % params + + return _URI_TEMPLATE_PLACEHOLDER.sub(substitute, uri_template) + class ItemFilter: """ @@ -514,7 +533,7 @@ class FeedExporter: self.slots.append( self._start_new_batch( batch_id=1, - uri=uri % uri_params, + uri=_format_uri_template(uri, uri_params), feed_options=feed_options, spider=spider, uri_template=uri, @@ -639,7 +658,7 @@ class FeedExporter: slots.append( self._start_new_batch( batch_id=slot.batch_id + 1, - uri=slot.uri_template % uri_params, + uri=_format_uri_template(slot.uri_template, uri_params), feed_options=self.feeds[slot.uri_template], spider=spider, uri_template=slot.uri_template, diff --git a/tests/test_feedexport_uri_params.py b/tests/test_feedexport_uri_params.py index 150d8449f..4ec78c8a0 100644 --- a/tests/test_feedexport_uri_params.py +++ b/tests/test_feedexport_uri_params.py @@ -7,7 +7,7 @@ import pytest import scrapy from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.extensions.feedexport import FeedExporter +from scrapy.extensions.feedexport import FeedExporter, _format_uri_template from scrapy.utils.test import get_crawler @@ -112,6 +112,70 @@ class TestURIParams(ABC): assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" + def test_percent_encoded_characters(self): + """Percent-encoded characters in the URI (e.g. in an FTP password) + must not be interpreted as printf-style format specifiers (#5794).""" + settings = self.build_settings( + uri="ftp://user:2%23um25%21M%23JZ@ftp.example.com/%(name)s.csv", + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + feed_exporter.open_spider(spider) + + assert ( + feed_exporter.slots[0].uri + == f"ftp://user:2%23um25%21M%23JZ@ftp.example.com/{self.spider_name}.csv" + ) + + +class TestFormatURITemplate: + params = { + "time": "2026-07-02T00-00-00", + "batch_id": 3, + "name": "myspider", + } + + @pytest.mark.parametrize( + ("template", "expected"), + [ + # percent-encoded characters are left untouched (#5794) + ( + "ftp://user:2%23um25%21M%23JZ@ftp.example.com/file.csv", + "ftp://user:2%23um25%21M%23JZ@ftp.example.com/file.csv", + ), + # named placeholders are substituted as before + ( + "s3://bucket/%(name)s/%(time)s.json", + "s3://bucket/myspider/2026-07-02T00-00-00.json", + ), + # conversion specs of named placeholders keep working + ( + "file:///tmp/batch-%(batch_id)05d.csv", + "file:///tmp/batch-00003.csv", + ), + # the %% printf escape keeps working + ( + "file:///tmp/100%%-%(name)s.csv", + "file:///tmp/100%-myspider.csv", + ), + # %25s is no longer read as a width-25 %s specifier, which used + # to silently inject the whole params dict into the URI + ( + "ftp://user:pa%25ss@host/file-%(name)s.csv", + "ftp://user:pa%25ss@host/file-myspider.csv", + ), + ], + ) + def test_substitution(self, template, expected): + assert _format_uri_template(template, self.params) == expected + + def test_missing_key_raises(self): + with pytest.raises(KeyError): + _format_uri_template("file:///tmp/%(missing)s.csv", self.params) + class TestURIParamsSetting(TestURIParams): deprecated_options = True