diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 6f54e62e9..1d3a11208 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -12,7 +12,6 @@ from twisted.internet.defer import ( inlineCallbacks, maybeDeferred, ) -from zope.interface.exceptions import DoesNotImplement try: # zope >= 5.0 only supports MultipleInvalid @@ -205,19 +204,7 @@ class CrawlerRunner: """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) - excs = ( - (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement - ) - try: - verifyClass(ISpiderLoader, loader_cls) - except excs: - warnings.warn( - "SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does " - "not fully implement scrapy.interfaces.ISpiderLoader interface. " - "Please add all missing methods to avoid unexpected runtime errors.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) + verifyClass(ISpiderLoader, loader_cls) return loader_cls.from_settings(settings.frozencopy()) def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None): diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py deleted file mode 100644 index 3b8702419..000000000 --- a/scrapy/downloadermiddlewares/decompression.py +++ /dev/null @@ -1,94 +0,0 @@ -""" This module implements the DecompressionMiddleware which tries to recognise -and extract the potentially compressed responses that may arrive. -""" - -import bz2 -import gzip -import logging -import tarfile -import zipfile -from io import BytesIO -from tempfile import mktemp -from warnings import warn - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.responsetypes import responsetypes - -warn( - "scrapy.downloadermiddlewares.decompression is deprecated", - ScrapyDeprecationWarning, - stacklevel=2, -) - - -logger = logging.getLogger(__name__) - - -class DecompressionMiddleware: - """This middleware tries to recognise and extract the possibly compressed - responses that may arrive.""" - - def __init__(self): - self._formats = { - "tar": self._is_tar, - "zip": self._is_zip, - "gz": self._is_gzip, - "bz2": self._is_bzip2, - } - - def _is_tar(self, response): - archive = BytesIO(response.body) - try: - tar_file = tarfile.open(name=mktemp(), fileobj=archive) - except tarfile.ReadError: - return - - body = tar_file.extractfile(tar_file.members[0]).read() - respcls = responsetypes.from_args(filename=tar_file.members[0].name, body=body) - return response.replace(body=body, cls=respcls) - - def _is_zip(self, response): - archive = BytesIO(response.body) - try: - zip_file = zipfile.ZipFile(archive) - except zipfile.BadZipFile: - return - - namelist = zip_file.namelist() - body = zip_file.read(namelist[0]) - respcls = responsetypes.from_args(filename=namelist[0], body=body) - return response.replace(body=body, cls=respcls) - - def _is_gzip(self, response): - archive = BytesIO(response.body) - try: - body = gzip.GzipFile(fileobj=archive).read() - except OSError: - return - - respcls = responsetypes.from_args(body=body) - return response.replace(body=body, cls=respcls) - - def _is_bzip2(self, response): - try: - body = bz2.decompress(response.body) - except OSError: - return - - respcls = responsetypes.from_args(body=body) - return response.replace(body=body, cls=respcls) - - def process_response(self, request, response, spider): - if not response.body: - return response - - for fmt, func in self._formats.items(): - new_response = func(response) - if new_response: - logger.debug( - "Decompressed response with format: %(responsefmt)s", - {"responsefmt": fmt}, - extra={"spider": spider}, - ) - return new_response - return response diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index de5a81388..5228db786 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -3,13 +3,10 @@ HTTP basic auth downloader middleware See documentation in docs/topics/downloader-middleware.rst """ -import warnings from w3lib.http import basic_auth_header from scrapy import signals -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.url import url_is_from_any_domain @@ -28,25 +25,10 @@ class HttpAuthMiddleware: pwd = getattr(spider, "http_pass", "") if usr or pwd: self.auth = basic_auth_header(usr, pwd) - if not hasattr(spider, "http_auth_domain"): - warnings.warn( - "Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security " - "problems if the spider makes requests to several different domains. http_auth_domain " - "will be set to the domain of the first request, please set it to the correct value " - "explicitly.", - category=ScrapyDeprecationWarning, - ) - self.domain_unset = True - else: - self.domain = spider.http_auth_domain - self.domain_unset = False + self.domain = spider.http_auth_domain def process_request(self, request, spider): auth = getattr(self, "auth", None) if auth and b"Authorization" not in request.headers: - domain = urlparse_cached(request).hostname - if self.domain_unset: - self.domain = domain - self.domain_unset = False if not self.domain or url_is_from_any_domain(request.url, [self.domain]): request.headers[b"Authorization"] = auth diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index ead426951..7b1d3f829 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,11 +1,9 @@ import io -import warnings import zlib from scrapy.exceptions import NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip ACCEPTED_ENCODINGS = [b"gzip", b"deflate"] @@ -36,18 +34,7 @@ class HttpCompressionMiddleware: def from_crawler(cls, crawler): if not crawler.settings.getbool("COMPRESSION_ENABLED"): raise NotConfigured - try: - return cls(stats=crawler.stats) - except TypeError: - warnings.warn( - "HttpCompressionMiddleware subclasses must either modify " - "their '__init__' method to support a 'stats' parameter or " - "reimplement the 'from_crawler' method.", - ScrapyDeprecationWarning, - ) - result = cls() - result.stats = crawler.stats - return result + return cls(stats=crawler.stats) def process_request(self, request, spider): request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 0b20f53b9..dd2420e98 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -3,14 +3,12 @@ from __future__ import annotations import logging from pathlib import Path from typing import TYPE_CHECKING, Optional, Set -from warnings import warn from twisted.internet.defer import Deferred from scrapy.http.request import Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.job import job_dir from scrapy.utils.request import ( RequestFingerprinter, @@ -75,38 +73,15 @@ class RFPDupeFilter(BaseDupeFilter): fingerprinter: Optional[RequestFingerprinterProtocol] = None, ) -> Self: debug = settings.getbool("DUPEFILTER_DEBUG") - try: - return cls(job_dir(settings), debug, fingerprinter=fingerprinter) - except TypeError: - warn( - "RFPDupeFilter subclasses must either modify their '__init__' " - "method to support a 'fingerprinter' parameter or reimplement " - "the 'from_settings' class method.", - ScrapyDeprecationWarning, - ) - result = cls(job_dir(settings), debug) - result.fingerprinter = fingerprinter or RequestFingerprinter() - return result + return cls(job_dir(settings), debug, fingerprinter=fingerprinter) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: assert crawler.request_fingerprinter - try: - return cls.from_settings( - crawler.settings, - fingerprinter=crawler.request_fingerprinter, - ) - except TypeError: - warn( - "RFPDupeFilter subclasses must either modify their overridden " - "'__init__' method and 'from_settings' class method to " - "support a 'fingerprinter' parameter, or reimplement the " - "'from_crawler' class method.", - ScrapyDeprecationWarning, - ) - result = cls.from_settings(crawler.settings) - result.fingerprinter = crawler.request_fingerprinter - return result + return cls.from_settings( + crawler.settings, + fingerprinter=crawler.request_fingerprinter, + ) def request_seen(self, request: Request) -> bool: fp = self.request_fingerprint(request) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 4e846d1bd..fadbbb582 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -29,7 +29,7 @@ from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.python import get_func_args, without_none_values +from scrapy.utils.python import without_none_values logger = logging.getLogger(__name__) @@ -42,17 +42,7 @@ except ImportError: def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): - argument_names = get_func_args(builder) - if "feed_options" in argument_names: - kwargs["feed_options"] = feed_options - else: - warnings.warn( - f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove this " - "warning. This parameter will become mandatory in a future " - "version of Scrapy.", - category=ScrapyDeprecationWarning, - ) + kwargs["feed_options"] = feed_options return builder(*preargs, uri, *args, **kwargs) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 0b443c7d0..06a52f5fd 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -200,7 +200,7 @@ class Request(object_ref): ``ignore_unknown_options=False``. .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request` - subclasses, such as :class:`~scrapy.http.JSONRequest`, or + subclasses, such as :class:`~scrapy.http.JsonRequest`, or :class:`~scrapy.http.XmlRpcRequest`, as well as having :ref:`downloader middlewares ` and diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 510c903db..f52c0401d 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -11,7 +11,6 @@ import warnings from typing import Optional, Tuple from scrapy.http.request import Request -from scrapy.utils.deprecate import create_deprecated_class class JsonRequest(Request): @@ -58,6 +57,3 @@ class JsonRequest(Request): def _dumps(self, data: dict) -> str: """Convert to JSON""" return json.dumps(data, **self._dumps_kwargs) - - -JSONRequest = create_deprecated_class("JSONRequest", JsonRequest) diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py deleted file mode 100644 index 3e8644e0c..000000000 --- a/scrapy/loader/common.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Common functions used in Item Loaders code""" - -import warnings - -from itemloaders import common - -from scrapy.utils.deprecate import ScrapyDeprecationWarning - - -def wrap_loader_context(function, context): - """Wrap functions that receive loader_context to contain the context - "pre-loaded" and expose a interface that receives only one argument - """ - warnings.warn( - "scrapy.loader.common.wrap_loader_context has moved to a new library." - "Please update your reference to itemloaders.common.wrap_loader_context", - ScrapyDeprecationWarning, - stacklevel=2, - ) - - return common.wrap_loader_context(function, context) diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py deleted file mode 100644 index b82c6d5c7..000000000 --- a/scrapy/loader/processors.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -This module provides some commonly used processors for Item Loaders. - -See documentation in docs/topics/loaders.rst -""" -from itemloaders import processors - -from scrapy.utils.deprecate import create_deprecated_class - -MapCompose = create_deprecated_class("MapCompose", processors.MapCompose) - -Compose = create_deprecated_class("Compose", processors.Compose) - -TakeFirst = create_deprecated_class("TakeFirst", processors.TakeFirst) - -Identity = create_deprecated_class("Identity", processors.Identity) - -SelectJmes = create_deprecated_class("SelectJmes", processors.SelectJmes) - -Join = create_deprecated_class("Join", processors.Join) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 401029439..75532034a 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,8 +1,6 @@ import functools import logging from collections import defaultdict -from inspect import signature -from warnings import warn from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure @@ -11,7 +9,6 @@ from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import defer_result, mustbe_deferred -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter @@ -44,9 +41,6 @@ class MediaPipeline: self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False) self._handle_statuses(self.allow_redirects) - # Check if deprecated methods are being used and make them compatible - self._make_compatible() - def _handle_statuses(self, allow_redirects): self.handle_httpstatus_list = None if allow_redirects: @@ -126,52 +120,6 @@ class MediaPipeline: ) return dfd.addBoth(lambda _: wad) # it must return wad at last - def _make_compatible(self): - """Make overridable methods of MediaPipeline and subclasses backwards compatible""" - methods = [ - "file_path", - "thumb_path", - "media_to_download", - "media_downloaded", - "file_downloaded", - "image_downloaded", - "get_images", - ] - - for method_name in methods: - method = getattr(self, method_name, None) - if callable(method): - setattr(self, method_name, self._compatible(method)) - - def _compatible(self, func): - """Wrapper for overridable methods to allow backwards compatibility""" - self._check_signature(func) - - @functools.wraps(func) - def wrapper(*args, **kwargs): - if self._expects_item[func.__name__]: - return func(*args, **kwargs) - - kwargs.pop("item", None) - return func(*args, **kwargs) - - return wrapper - - def _check_signature(self, func): - sig = signature(func) - self._expects_item[func.__name__] = True - - if "item" not in sig.parameters: - old_params = str(sig)[1:-1] - new_params = old_params + ", *, item=None" - warn( - f"{func.__name__}(self, {old_params}) is deprecated, " - f"please use {func.__name__}(self, {new_params})", - ScrapyDeprecationWarning, - stacklevel=2, - ) - self._expects_item[func.__name__] = False - def _modify_media_request(self, request): if self.handle_httpstatus_list: request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b3c28da92..a9364bea2 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -21,17 +21,12 @@ from typing import ( Iterable, List, Optional, - Pattern, Union, cast, ) -from w3lib.html import replace_entities - from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache -from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.python import flatten, to_unicode if TYPE_CHECKING: from scrapy import Spider @@ -108,39 +103,6 @@ def walk_modules(path: str) -> List[ModuleType]: return mods -def extract_regex( - regex: Union[str, Pattern], text: str, encoding: str = "utf-8" -) -> List[str]: - """Extract a list of unicode strings from the given text/encoding using the following policies: - - * if the regex contains a named group called "extract" that will be returned - * if the regex contains multiple numbered groups, all those will be returned (flattened) - * if the regex doesn't contain any group the entire regex matching is returned - """ - warnings.warn( - "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - - if isinstance(regex, str): - regex = re.compile(regex, re.UNICODE) - - try: - # named group - strings = [regex.search(text).group("extract")] # type: ignore[union-attr] - except Exception: - # full regex or numbered groups - strings = regex.findall(text) - strings = flatten(strings) - - if isinstance(text, str): - return [replace_entities(s, keep=["lt", "amp"]) for s in strings] - return [ - replace_entities(to_unicode(s, encoding), keep=["lt", "amp"]) for s in strings - ] - - def md5sum(file: IO) -> str: """Calculate the md5 checksum of a file-like object without reading its whole content in memory. diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 60b92377d..a71b9c2b7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -15,6 +15,7 @@ from pytest import mark, raises from twisted.internet import defer from twisted.trial import unittest from w3lib import __version__ as w3lib_version +from zope.interface.exceptions import MultipleInvalid import scrapy from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner @@ -179,11 +180,7 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, } ) - with warnings.catch_warnings(record=True) as w: - self.assertRaises(AttributeError, CrawlerRunner, settings) - self.assertEqual(len(w), 1) - self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message)) - self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message)) + self.assertRaises(MultipleInvalid, CrawlerRunner, settings) def test_crawler_runner_accepts_dict(self): runner = CrawlerRunner({"foo": "bar"}) diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py deleted file mode 100644 index 95739414e..000000000 --- a/tests/test_downloadermiddleware_decompression.py +++ /dev/null @@ -1,53 +0,0 @@ -from unittest import TestCase, main - -from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware -from scrapy.http import Response, XmlResponse -from scrapy.spiders import Spider -from scrapy.utils.test import assert_samelines -from tests import get_testdata - - -def _test_data(formats): - uncompressed_body = get_testdata("compressed", "feed-sample1.xml") - test_responses = {} - for format in formats: - body = get_testdata("compressed", "feed-sample1." + format) - test_responses[format] = Response("http://foo.com/bar", body=body) - return uncompressed_body, test_responses - - -class DecompressionMiddlewareTest(TestCase): - test_formats = ["tar", "xml.bz2", "xml.gz", "zip"] - uncompressed_body, test_responses = _test_data(test_formats) - - def setUp(self): - self.mw = DecompressionMiddleware() - self.spider = Spider("foo") - - def test_known_compression_formats(self): - for fmt in self.test_formats: - rsp = self.test_responses[fmt] - new = self.mw.process_response(None, rsp, self.spider) - error_msg = f"Failed {fmt}, response type {type(new).__name__}" - assert isinstance(new, XmlResponse), error_msg - assert_samelines(self, new.body, self.uncompressed_body, fmt) - - def test_plain_response(self): - rsp = Response(url="http://test.com", body=self.uncompressed_body) - new = self.mw.process_response(None, rsp, self.spider) - assert new is rsp - assert_samelines(self, new.body, rsp.body) - - def test_empty_response(self): - rsp = Response(url="http://test.com", body=b"") - new = self.mw.process_response(None, rsp, self.spider) - assert new is rsp - assert not rsp.body - assert not new.body - - def tearDown(self): - del self.mw - - -if __name__ == "__main__": - main() diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 6b79234d0..fc110e6cc 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,10 +1,8 @@ import unittest -import pytest from w3lib.http import basic_auth_header from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.spiders import Spider @@ -31,39 +29,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): self.spider = TestSpiderLegacy("foo") def test_auth(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Using HttpAuthMiddleware without http_auth_domain is deprecated", - ): + with self.assertRaises(AttributeError): mw = HttpAuthMiddleware() mw.spider_opened(self.spider) - # initial request, sets the domain and sends the header - req = Request("http://example.com/") - assert mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) - - # subsequent request to the same domain, should send the header - req = Request("http://example.com/") - assert mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) - - # subsequent request to a different domain, shouldn't send the header - req = Request("http://example-noauth.com/") - assert mw.process_request(req, self.spider) is None - self.assertNotIn("Authorization", req.headers) - - def test_auth_already_set(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Using HttpAuthMiddleware without http_auth_domain is deprecated", - ): - mw = HttpAuthMiddleware() - mw.spider_opened(self.spider) - req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) - assert mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], b"Digest 123") - class HttpAuthMiddlewareTest(unittest.TestCase): def setUp(self): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 9dad056de..a96b710f3 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -2,7 +2,6 @@ from gzip import GzipFile from io import BytesIO from pathlib import Path from unittest import SkipTest, TestCase -from warnings import catch_warnings from w3lib.encoding import resolve_encoding @@ -10,7 +9,7 @@ from scrapy.downloadermiddlewares.httpcompression import ( ACCEPTED_ENCODINGS, HttpCompressionMiddleware, ) -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider @@ -372,29 +371,3 @@ class HttpCompressionTest(TestCase): self.assertEqual(response.body, b"") self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_bytes", None) - - -class HttpCompressionSubclassTest(TestCase): - def test_init_missing_stats(self): - class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware): - def __init__(self): - super().__init__() - - crawler = get_crawler(Spider) - with catch_warnings(record=True) as caught_warnings: - HttpCompressionMiddlewareSubclass.from_crawler(crawler) - messages = tuple( - str(warning.message) - for warning in caught_warnings - if warning.category is ScrapyDeprecationWarning - ) - self.assertEqual( - messages, - ( - ( - "HttpCompressionMiddleware subclasses must either modify " - "their '__init__' method to support a 'stats' parameter " - "or reimplement the 'from_crawler' method." - ), - ), - ) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 56967c0d5..89169fd7c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2835,200 +2835,6 @@ class FeedExportInitTest(unittest.TestCase): self.assertIsInstance(exporter, FeedExporter) -class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage): - def __init__(self, uri): - super().__init__(uri) - - -class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): - """Make sure that any feed exporter created by users before the - introduction of the ``feed_options`` parameter continues to work as - expected, and simply issues a warning.""" - - def test_init(self): - settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": StdoutFeedStorageWithoutFeedOptions}, - } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - with pytest.warns( - ScrapyDeprecationWarning, - match="StdoutFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) - - -class FileFeedStorageWithoutFeedOptions(FileFeedStorage): - def __init__(self, uri): - super().__init__(uri) - - -class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): - """Make sure that any feed exporter created by users before the - introduction of the ``feed_options`` parameter continues to work as - expected, and simply issues a warning.""" - - maxDiff = None - - def test_init(self): - with tempfile.NamedTemporaryFile() as temp: - settings_dict = { - "FEED_URI": f"file:///{temp.name}", - "FEED_STORAGES": {"file": FileFeedStorageWithoutFeedOptions}, - } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - spider = scrapy.Spider("default") - - with pytest.warns( - ScrapyDeprecationWarning, - match="FileFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) - - -class S3FeedStorageWithoutFeedOptions(S3FeedStorage): - def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs): - super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs) - - -class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): - @classmethod - def from_crawler(cls, crawler, uri): - return super().from_crawler(crawler, uri) - - -class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): - """Make sure that any feed exporter created by users before the - introduction of the ``feed_options`` parameter continues to work as - expected, and simply issues a warning.""" - - maxDiff = None - - def setUp(self): - skip_if_no_boto() - - def test_init(self): - settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptions}, - } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - spider.crawler = crawler - - with pytest.warns( - ScrapyDeprecationWarning, - match="S3FeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) - - def test_from_crawler(self): - settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptionsWithFromCrawler}, - } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - spider.crawler = crawler - - with pytest.warns( - ScrapyDeprecationWarning, - match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) - - -class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage): - def __init__(self, uri, use_active_mode=False): - super().__init__(uri) - - -class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage): - @classmethod - def from_crawler(cls, crawler, uri): - return super().from_crawler(crawler, uri) - - -class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): - """Make sure that any feed exporter created by users before the - introduction of the ``feed_options`` parameter continues to work as - expected, and simply issues a warning.""" - - maxDiff = None - - def test_init(self): - settings_dict = { - "FEED_URI": "ftp://localhost/foo", - "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptions}, - } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - spider.crawler = crawler - - with pytest.warns( - ScrapyDeprecationWarning, - match="FTPFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) - - def test_from_crawler(self): - settings_dict = { - "FEED_URI": "ftp://localhost/foo", - "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptionsWithFromCrawler}, - } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - spider.crawler = crawler - - with pytest.warns( - ScrapyDeprecationWarning, - match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) - - class URIParamsTest: spider_name = "uri_params_spider" deprecated_options = False diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 3eae87599..d7f773d5c 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -4,7 +4,6 @@ Once we remove the references from scrapy, we can remove these tests. """ import unittest -import warnings from functools import partial from itemloaders.processors import ( @@ -18,9 +17,6 @@ from itemloaders.processors import ( from scrapy.item import Field, Item from scrapy.loader import ItemLoader -from scrapy.loader.common import wrap_loader_context -from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.misc import extract_regex # test items @@ -722,24 +718,5 @@ class FunctionProcessorTestCase(unittest.TestCase): self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) -class DeprecatedUtilityFunctionsTestCase(unittest.TestCase): - def test_deprecated_wrap_loader_context(self): - def function(*args): - return None - - with warnings.catch_warnings(record=True) as w: - wrap_loader_context(function, context={}) - - assert len(w) == 1 - assert issubclass(w[0].category, ScrapyDeprecationWarning) - - def test_deprecated_extract_regex(self): - with warnings.catch_warnings(record=True) as w: - extract_regex(r"\w+", "this is a test") - - assert len(w) == 1 - assert issubclass(w[0].category, ScrapyDeprecationWarning) - - if __name__ == "__main__": unittest.main() diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d655eb128..820484565 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -15,7 +15,6 @@ from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.spiders import Spider -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler @@ -427,102 +426,6 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): return super().image_downloaded(response, request, info) -class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): - skip = skip_pillow - - def setUp(self): - settings_dict = { - "IMAGES_STORE": "store-uri", - "IMAGES_THUMBS": {"small": (50, 50)}, - } - crawler = get_crawler(spidercls=None, settings_dict=settings_dict) - self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler) - self.pipe.download_func = _mocked_download_func - self.pipe.open_spider(None) - self.item = dict(image_urls=["http://picsum.photos/id/1014/200/300"], images=[]) - - def _assert_method_called_with_warnings(self, method, message, warnings): - self.assertIn(method, self.pipe._mockcalled) - warningShown = False - for warning in warnings: - if ( - warning["message"] == message - and warning["category"] == ScrapyDeprecationWarning - ): - warningShown = True - self.assertTrue(warningShown) - - @inlineCallbacks - def test_media_to_download_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "media_to_download(self, request, info) is deprecated, " - "please use media_to_download(self, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("media_to_download", message, warnings) - - @inlineCallbacks - def test_media_downloaded_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "media_downloaded(self, response, request, info) is deprecated, " - "please use media_downloaded(self, response, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("media_downloaded", message, warnings) - - @inlineCallbacks - def test_file_downloaded_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "file_downloaded(self, response, request, info) is deprecated, " - "please use file_downloaded(self, response, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("file_downloaded", message, warnings) - - @inlineCallbacks - def test_file_path_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "file_path(self, request, response=None, info=None) is deprecated, " - "please use file_path(self, request, response=None, info=None, *, item=None)" - ) - self._assert_method_called_with_warnings("file_path", message, warnings) - - @inlineCallbacks - def test_thumb_path_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, " - "please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)" - ) - self._assert_method_called_with_warnings("thumb_path", message, warnings) - - @inlineCallbacks - def test_get_images_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "get_images(self, response, request, info) is deprecated, " - "please use get_images(self, response, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("get_images", message, warnings) - - @inlineCallbacks - def test_image_downloaded_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "image_downloaded(self, response, request, info) is deprecated, " - "please use image_downloaded(self, response, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("image_downloaded", message, warnings) - - class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): def _assert_request_no3xx(self, pipeline_class, settings): pipe = pipeline_class(settings=Settings(settings))