RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove support for feed storage backends without feed_options

This commit is contained in:
Laerte Pereira 2023-10-17 22:39:09 -03:00
parent 3db438127c
commit 644ab3af48
2 changed files with 11 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
scrapy/extensions/feedexport.py
View File

@ -29,7 +29,7 @@ from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import get_func_args, without_none_values
from scrapy.utils.python import without_none_values
logger = logging.getLogger(__name__)
@ -42,17 +42,7 @@ except ImportError:
def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
argument_names = get_func_args(builder)
if "feed_options" in argument_names:
kwargs["feed_options"] = feed_options
else:
warnings.warn(
f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a "
"'feed_options' parameter to its signature to remove this "
"warning. This parameter will become mandatory in a future "
"version of Scrapy.",
category=ScrapyDeprecationWarning,
)
kwargs["feed_options"] = feed_options
return builder(*preargs, uri, *args, **kwargs)

115
tests/test_feedexport.py
View File

@ -2850,20 +2850,8 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):
"FEED_URI": "file:///tmp/foobar",
"FEED_STORAGES": {"file": StdoutFeedStorageWithoutFeedOptions},
}
with pytest.warns(
ScrapyDeprecationWarning,
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
):
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
with pytest.warns(
ScrapyDeprecationWarning,
match="StdoutFeedStorageWithoutFeedOptions does not support "
"the 'feed_options' keyword argument.",
):
feed_exporter.open_spider(spider)
with pytest.raises(TypeError):
get_crawler(settings_dict=settings_dict)
class FileFeedStorageWithoutFeedOptions(FileFeedStorage):
@ -2872,10 +2860,6 @@ class FileFeedStorageWithoutFeedOptions(FileFeedStorage):
class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
"""Make sure that any feed exporter created by users before the
introduction of the ``feed_options`` parameter continues to work as
expected, and simply issues a warning."""
maxDiff = None
def test_init(self):
@ -2884,20 +2868,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
"FEED_URI": f"file:///{temp.name}",
"FEED_STORAGES": {"file": FileFeedStorageWithoutFeedOptions},
}
with pytest.warns(
ScrapyDeprecationWarning,
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
):
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
with pytest.warns(
ScrapyDeprecationWarning,
match="FileFeedStorageWithoutFeedOptions does not support "
"the 'feed_options' keyword argument.",
):
feed_exporter.open_spider(spider)
with self.assertRaises(TypeError):
get_crawler(settings_dict=settings_dict)
class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
@ -2912,10 +2884,6 @@ class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):
class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
"""Make sure that any feed exporter created by users before the
introduction of the ``feed_options`` parameter continues to work as
expected, and simply issues a warning."""
maxDiff = None
def setUp(self):
@ -2936,34 +2904,15 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
spider = scrapy.Spider("default")
spider.crawler = crawler
with pytest.warns(
ScrapyDeprecationWarning,
match="S3FeedStorageWithoutFeedOptions does not support "
"the 'feed_options' keyword argument.",
):
feed_exporter.open_spider(spider)
feed_exporter.open_spider(spider)
def test_from_crawler(self):
settings_dict = {
"FEED_URI": "file:///tmp/foobar",
"FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptionsWithFromCrawler},
}
with pytest.warns(
ScrapyDeprecationWarning,
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
):
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
spider.crawler = crawler
with pytest.warns(
ScrapyDeprecationWarning,
match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
"the 'feed_options' keyword argument.",
):
feed_exporter.open_spider(spider)
with pytest.raises(TypeError):
get_crawler(settings_dict=settings_dict)
class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):
@ -2971,17 +2920,7 @@ class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):
super().__init__(uri)
class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage):
@classmethod
def from_crawler(cls, crawler, uri):
return super().from_crawler(crawler, uri)
class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
"""Make sure that any feed exporter created by users before the
introduction of the ``feed_options`` parameter continues to work as
expected, and simply issues a warning."""
maxDiff = None
def test_init(self):
@ -2989,44 +2928,8 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
"FEED_URI": "ftp://localhost/foo",
"FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptions},
}
with pytest.warns(
ScrapyDeprecationWarning,
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
):
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
spider.crawler = crawler
with pytest.warns(
ScrapyDeprecationWarning,
match="FTPFeedStorageWithoutFeedOptions does not support "
"the 'feed_options' keyword argument.",
):
feed_exporter.open_spider(spider)
def test_from_crawler(self):
settings_dict = {
"FEED_URI": "ftp://localhost/foo",
"FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptionsWithFromCrawler},
}
with pytest.warns(
ScrapyDeprecationWarning,
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
):
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
spider.crawler = crawler
with pytest.warns(
ScrapyDeprecationWarning,
match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
"the 'feed_options' keyword argument.",
):
feed_exporter.open_spider(spider)
with pytest.raises(TypeError):
get_crawler(settings_dict=settings_dict)
class URIParamsTest: