mirror of https://github.com/scrapy/scrapy.git
Merge pull request #6113 from Laerte/master
Remove some deprecated code
This commit is contained in:
Adrián Chaves
GitHub
commit
9b06f6b316
|
|
@ -12,7 +12,6 @@ from twisted.internet.defer import (
|
|||
inlineCallbacks,
|
||||
maybeDeferred,
|
||||
)
|
||||
from zope.interface.exceptions import DoesNotImplement
|
||||
|
||||
try:
|
||||
# zope >= 5.0 only supports MultipleInvalid
|
||||
|
|
@ -205,19 +204,7 @@ class CrawlerRunner:
|
|||
"""Get SpiderLoader instance from settings"""
|
||||
cls_path = settings.get("SPIDER_LOADER_CLASS")
|
||||
loader_cls = load_object(cls_path)
|
||||
excs = (
|
||||
(DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement
|
||||
)
|
||||
try:
|
||||
verifyClass(ISpiderLoader, loader_cls)
|
||||
except excs:
|
||||
warnings.warn(
|
||||
"SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does "
|
||||
"not fully implement scrapy.interfaces.ISpiderLoader interface. "
|
||||
"Please add all missing methods to avoid unexpected runtime errors.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
verifyClass(ISpiderLoader, loader_cls)
|
||||
return loader_cls.from_settings(settings.frozencopy())
|
||||
|
||||
def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None):
|
||||
|
|
|
|||
|
|
@ -1,94 +0,0 @@
|
|||
""" This module implements the DecompressionMiddleware which tries to recognise
|
||||
and extract the potentially compressed responses that may arrive.
|
||||
"""
|
||||
|
||||
import bz2
|
||||
import gzip
|
||||
import logging
|
||||
import tarfile
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
from tempfile import mktemp
|
||||
from warnings import warn
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.responsetypes import responsetypes
|
||||
|
||||
warn(
|
||||
"scrapy.downloadermiddlewares.decompression is deprecated",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DecompressionMiddleware:
|
||||
"""This middleware tries to recognise and extract the possibly compressed
|
||||
responses that may arrive."""
|
||||
|
||||
def __init__(self):
|
||||
self._formats = {
|
||||
"tar": self._is_tar,
|
||||
"zip": self._is_zip,
|
||||
"gz": self._is_gzip,
|
||||
"bz2": self._is_bzip2,
|
||||
}
|
||||
|
||||
def _is_tar(self, response):
|
||||
archive = BytesIO(response.body)
|
||||
try:
|
||||
tar_file = tarfile.open(name=mktemp(), fileobj=archive)
|
||||
except tarfile.ReadError:
|
||||
return
|
||||
|
||||
body = tar_file.extractfile(tar_file.members[0]).read()
|
||||
respcls = responsetypes.from_args(filename=tar_file.members[0].name, body=body)
|
||||
return response.replace(body=body, cls=respcls)
|
||||
|
||||
def _is_zip(self, response):
|
||||
archive = BytesIO(response.body)
|
||||
try:
|
||||
zip_file = zipfile.ZipFile(archive)
|
||||
except zipfile.BadZipFile:
|
||||
return
|
||||
|
||||
namelist = zip_file.namelist()
|
||||
body = zip_file.read(namelist[0])
|
||||
respcls = responsetypes.from_args(filename=namelist[0], body=body)
|
||||
return response.replace(body=body, cls=respcls)
|
||||
|
||||
def _is_gzip(self, response):
|
||||
archive = BytesIO(response.body)
|
||||
try:
|
||||
body = gzip.GzipFile(fileobj=archive).read()
|
||||
except OSError:
|
||||
return
|
||||
|
||||
respcls = responsetypes.from_args(body=body)
|
||||
return response.replace(body=body, cls=respcls)
|
||||
|
||||
def _is_bzip2(self, response):
|
||||
try:
|
||||
body = bz2.decompress(response.body)
|
||||
except OSError:
|
||||
return
|
||||
|
||||
respcls = responsetypes.from_args(body=body)
|
||||
return response.replace(body=body, cls=respcls)
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if not response.body:
|
||||
return response
|
||||
|
||||
for fmt, func in self._formats.items():
|
||||
new_response = func(response)
|
||||
if new_response:
|
||||
logger.debug(
|
||||
"Decompressed response with format: %(responsefmt)s",
|
||||
{"responsefmt": fmt},
|
||||
extra={"spider": spider},
|
||||
)
|
||||
return new_response
|
||||
return response
|
||||
|
|
@ -3,13 +3,10 @@ HTTP basic auth downloader middleware
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
import warnings
|
||||
|
||||
from w3lib.http import basic_auth_header
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.url import url_is_from_any_domain
|
||||
|
||||
|
||||
|
|
@ -28,25 +25,10 @@ class HttpAuthMiddleware:
|
|||
pwd = getattr(spider, "http_pass", "")
|
||||
if usr or pwd:
|
||||
self.auth = basic_auth_header(usr, pwd)
|
||||
if not hasattr(spider, "http_auth_domain"):
|
||||
warnings.warn(
|
||||
"Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security "
|
||||
"problems if the spider makes requests to several different domains. http_auth_domain "
|
||||
"will be set to the domain of the first request, please set it to the correct value "
|
||||
"explicitly.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
)
|
||||
self.domain_unset = True
|
||||
else:
|
||||
self.domain = spider.http_auth_domain
|
||||
self.domain_unset = False
|
||||
self.domain = spider.http_auth_domain
|
||||
|
||||
def process_request(self, request, spider):
|
||||
auth = getattr(self, "auth", None)
|
||||
if auth and b"Authorization" not in request.headers:
|
||||
domain = urlparse_cached(request).hostname
|
||||
if self.domain_unset:
|
||||
self.domain = domain
|
||||
self.domain_unset = False
|
||||
if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
|
||||
request.headers[b"Authorization"] = auth
|
||||
|
|
|
|||
|
|
@ -1,11 +1,9 @@
|
|||
import io
|
||||
import warnings
|
||||
import zlib
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response, TextResponse
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.gz import gunzip
|
||||
|
||||
ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]
|
||||
|
|
@ -36,18 +34,7 @@ class HttpCompressionMiddleware:
|
|||
def from_crawler(cls, crawler):
|
||||
if not crawler.settings.getbool("COMPRESSION_ENABLED"):
|
||||
raise NotConfigured
|
||||
try:
|
||||
return cls(stats=crawler.stats)
|
||||
except TypeError:
|
||||
warnings.warn(
|
||||
"HttpCompressionMiddleware subclasses must either modify "
|
||||
"their '__init__' method to support a 'stats' parameter or "
|
||||
"reimplement the 'from_crawler' method.",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
result = cls()
|
||||
result.stats = crawler.stats
|
||||
return result
|
||||
return cls(stats=crawler.stats)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
|
||||
|
|
|
|||
|
|
@ -3,14 +3,12 @@ from __future__ import annotations
|
|||
import logging
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Optional, Set
|
||||
from warnings import warn
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.settings import BaseSettings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.job import job_dir
|
||||
from scrapy.utils.request import (
|
||||
RequestFingerprinter,
|
||||
|
|
@ -75,38 +73,15 @@ class RFPDupeFilter(BaseDupeFilter):
|
|||
fingerprinter: Optional[RequestFingerprinterProtocol] = None,
|
||||
) -> Self:
|
||||
debug = settings.getbool("DUPEFILTER_DEBUG")
|
||||
try:
|
||||
return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
|
||||
except TypeError:
|
||||
warn(
|
||||
"RFPDupeFilter subclasses must either modify their '__init__' "
|
||||
"method to support a 'fingerprinter' parameter or reimplement "
|
||||
"the 'from_settings' class method.",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
result = cls(job_dir(settings), debug)
|
||||
result.fingerprinter = fingerprinter or RequestFingerprinter()
|
||||
return result
|
||||
return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
assert crawler.request_fingerprinter
|
||||
try:
|
||||
return cls.from_settings(
|
||||
crawler.settings,
|
||||
fingerprinter=crawler.request_fingerprinter,
|
||||
)
|
||||
except TypeError:
|
||||
warn(
|
||||
"RFPDupeFilter subclasses must either modify their overridden "
|
||||
"'__init__' method and 'from_settings' class method to "
|
||||
"support a 'fingerprinter' parameter, or reimplement the "
|
||||
"'from_crawler' class method.",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
result = cls.from_settings(crawler.settings)
|
||||
result.fingerprinter = crawler.request_fingerprinter
|
||||
return result
|
||||
return cls.from_settings(
|
||||
crawler.settings,
|
||||
fingerprinter=crawler.request_fingerprinter,
|
||||
)
|
||||
|
||||
def request_seen(self, request: Request) -> bool:
|
||||
fp = self.request_fingerprint(request)
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from scrapy.utils.deprecate import create_deprecated_class
|
|||
from scrapy.utils.ftp import ftp_store_file
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.python import get_func_args, without_none_values
|
||||
from scrapy.utils.python import without_none_values
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -42,17 +42,7 @@ except ImportError:
|
|||
|
||||
|
||||
def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
|
||||
argument_names = get_func_args(builder)
|
||||
if "feed_options" in argument_names:
|
||||
kwargs["feed_options"] = feed_options
|
||||
else:
|
||||
warnings.warn(
|
||||
f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a "
|
||||
"'feed_options' parameter to its signature to remove this "
|
||||
"warning. This parameter will become mandatory in a future "
|
||||
"version of Scrapy.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
)
|
||||
kwargs["feed_options"] = feed_options
|
||||
return builder(*preargs, uri, *args, **kwargs)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -200,7 +200,7 @@ class Request(object_ref):
|
|||
``ignore_unknown_options=False``.
|
||||
|
||||
.. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request`
|
||||
subclasses, such as :class:`~scrapy.http.JSONRequest`, or
|
||||
subclasses, such as :class:`~scrapy.http.JsonRequest`, or
|
||||
:class:`~scrapy.http.XmlRpcRequest`, as well as having
|
||||
:ref:`downloader middlewares <topics-downloader-middleware>`
|
||||
and
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ import warnings
|
|||
from typing import Optional, Tuple
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.utils.deprecate import create_deprecated_class
|
||||
|
||||
|
||||
class JsonRequest(Request):
|
||||
|
|
@ -58,6 +57,3 @@ class JsonRequest(Request):
|
|||
def _dumps(self, data: dict) -> str:
|
||||
"""Convert to JSON"""
|
||||
return json.dumps(data, **self._dumps_kwargs)
|
||||
|
||||
|
||||
JSONRequest = create_deprecated_class("JSONRequest", JsonRequest)
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
"""Common functions used in Item Loaders code"""
|
||||
|
||||
import warnings
|
||||
|
||||
from itemloaders import common
|
||||
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
def wrap_loader_context(function, context):
|
||||
"""Wrap functions that receive loader_context to contain the context
|
||||
"pre-loaded" and expose a interface that receives only one argument
|
||||
"""
|
||||
warnings.warn(
|
||||
"scrapy.loader.common.wrap_loader_context has moved to a new library."
|
||||
"Please update your reference to itemloaders.common.wrap_loader_context",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
return common.wrap_loader_context(function, context)
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
"""
|
||||
This module provides some commonly used processors for Item Loaders.
|
||||
|
||||
See documentation in docs/topics/loaders.rst
|
||||
"""
|
||||
from itemloaders import processors
|
||||
|
||||
from scrapy.utils.deprecate import create_deprecated_class
|
||||
|
||||
MapCompose = create_deprecated_class("MapCompose", processors.MapCompose)
|
||||
|
||||
Compose = create_deprecated_class("Compose", processors.Compose)
|
||||
|
||||
TakeFirst = create_deprecated_class("TakeFirst", processors.TakeFirst)
|
||||
|
||||
Identity = create_deprecated_class("Identity", processors.Identity)
|
||||
|
||||
SelectJmes = create_deprecated_class("SelectJmes", processors.SelectJmes)
|
||||
|
||||
Join = create_deprecated_class("Join", processors.Join)
|
||||
|
|
@ -1,8 +1,6 @@
|
|||
import functools
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from inspect import signature
|
||||
from warnings import warn
|
||||
|
||||
from twisted.internet.defer import Deferred, DeferredList
|
||||
from twisted.python.failure import Failure
|
||||
|
|
@ -11,7 +9,6 @@ from scrapy.http.request import NO_CALLBACK
|
|||
from scrapy.settings import Settings
|
||||
from scrapy.utils.datatypes import SequenceExclude
|
||||
from scrapy.utils.defer import defer_result, mustbe_deferred
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
|
||||
|
|
@ -44,9 +41,6 @@ class MediaPipeline:
|
|||
self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False)
|
||||
self._handle_statuses(self.allow_redirects)
|
||||
|
||||
# Check if deprecated methods are being used and make them compatible
|
||||
self._make_compatible()
|
||||
|
||||
def _handle_statuses(self, allow_redirects):
|
||||
self.handle_httpstatus_list = None
|
||||
if allow_redirects:
|
||||
|
|
@ -126,52 +120,6 @@ class MediaPipeline:
|
|||
)
|
||||
return dfd.addBoth(lambda _: wad) # it must return wad at last
|
||||
|
||||
def _make_compatible(self):
|
||||
"""Make overridable methods of MediaPipeline and subclasses backwards compatible"""
|
||||
methods = [
|
||||
"file_path",
|
||||
"thumb_path",
|
||||
"media_to_download",
|
||||
"media_downloaded",
|
||||
"file_downloaded",
|
||||
"image_downloaded",
|
||||
"get_images",
|
||||
]
|
||||
|
||||
for method_name in methods:
|
||||
method = getattr(self, method_name, None)
|
||||
if callable(method):
|
||||
setattr(self, method_name, self._compatible(method))
|
||||
|
||||
def _compatible(self, func):
|
||||
"""Wrapper for overridable methods to allow backwards compatibility"""
|
||||
self._check_signature(func)
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
if self._expects_item[func.__name__]:
|
||||
return func(*args, **kwargs)
|
||||
|
||||
kwargs.pop("item", None)
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
def _check_signature(self, func):
|
||||
sig = signature(func)
|
||||
self._expects_item[func.__name__] = True
|
||||
|
||||
if "item" not in sig.parameters:
|
||||
old_params = str(sig)[1:-1]
|
||||
new_params = old_params + ", *, item=None"
|
||||
warn(
|
||||
f"{func.__name__}(self, {old_params}) is deprecated, "
|
||||
f"please use {func.__name__}(self, {new_params})",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
self._expects_item[func.__name__] = False
|
||||
|
||||
def _modify_media_request(self, request):
|
||||
if self.handle_httpstatus_list:
|
||||
request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list
|
||||
|
|
|
|||
|
|
@ -21,17 +21,12 @@ from typing import (
|
|||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Pattern,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
from w3lib.html import replace_entities
|
||||
|
||||
from scrapy.item import Item
|
||||
from scrapy.utils.datatypes import LocalWeakReferencedCache
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.python import flatten, to_unicode
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy import Spider
|
||||
|
|
@ -108,39 +103,6 @@ def walk_modules(path: str) -> List[ModuleType]:
|
|||
return mods
|
||||
|
||||
|
||||
def extract_regex(
|
||||
regex: Union[str, Pattern], text: str, encoding: str = "utf-8"
|
||||
) -> List[str]:
|
||||
"""Extract a list of unicode strings from the given text/encoding using the following policies:
|
||||
|
||||
* if the regex contains a named group called "extract" that will be returned
|
||||
* if the regex contains multiple numbered groups, all those will be returned (flattened)
|
||||
* if the regex doesn't contain any group the entire regex matching is returned
|
||||
"""
|
||||
warnings.warn(
|
||||
"scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
if isinstance(regex, str):
|
||||
regex = re.compile(regex, re.UNICODE)
|
||||
|
||||
try:
|
||||
# named group
|
||||
strings = [regex.search(text).group("extract")] # type: ignore[union-attr]
|
||||
except Exception:
|
||||
# full regex or numbered groups
|
||||
strings = regex.findall(text)
|
||||
strings = flatten(strings)
|
||||
|
||||
if isinstance(text, str):
|
||||
return [replace_entities(s, keep=["lt", "amp"]) for s in strings]
|
||||
return [
|
||||
replace_entities(to_unicode(s, encoding), keep=["lt", "amp"]) for s in strings
|
||||
]
|
||||
|
||||
|
||||
def md5sum(file: IO) -> str:
|
||||
"""Calculate the md5 checksum of a file-like object without reading its
|
||||
whole content in memory.
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from pytest import mark, raises
|
|||
from twisted.internet import defer
|
||||
from twisted.trial import unittest
|
||||
from w3lib import __version__ as w3lib_version
|
||||
from zope.interface.exceptions import MultipleInvalid
|
||||
|
||||
import scrapy
|
||||
from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner
|
||||
|
|
@ -179,11 +180,7 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
|
|||
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
|
||||
}
|
||||
)
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
self.assertRaises(AttributeError, CrawlerRunner, settings)
|
||||
self.assertEqual(len(w), 1)
|
||||
self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message))
|
||||
self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message))
|
||||
self.assertRaises(MultipleInvalid, CrawlerRunner, settings)
|
||||
|
||||
def test_crawler_runner_accepts_dict(self):
|
||||
runner = CrawlerRunner({"foo": "bar"})
|
||||
|
|
|
|||
|
|
@ -1,53 +0,0 @@
|
|||
from unittest import TestCase, main
|
||||
|
||||
from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware
|
||||
from scrapy.http import Response, XmlResponse
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.test import assert_samelines
|
||||
from tests import get_testdata
|
||||
|
||||
|
||||
def _test_data(formats):
|
||||
uncompressed_body = get_testdata("compressed", "feed-sample1.xml")
|
||||
test_responses = {}
|
||||
for format in formats:
|
||||
body = get_testdata("compressed", "feed-sample1." + format)
|
||||
test_responses[format] = Response("http://foo.com/bar", body=body)
|
||||
return uncompressed_body, test_responses
|
||||
|
||||
|
||||
class DecompressionMiddlewareTest(TestCase):
|
||||
test_formats = ["tar", "xml.bz2", "xml.gz", "zip"]
|
||||
uncompressed_body, test_responses = _test_data(test_formats)
|
||||
|
||||
def setUp(self):
|
||||
self.mw = DecompressionMiddleware()
|
||||
self.spider = Spider("foo")
|
||||
|
||||
def test_known_compression_formats(self):
|
||||
for fmt in self.test_formats:
|
||||
rsp = self.test_responses[fmt]
|
||||
new = self.mw.process_response(None, rsp, self.spider)
|
||||
error_msg = f"Failed {fmt}, response type {type(new).__name__}"
|
||||
assert isinstance(new, XmlResponse), error_msg
|
||||
assert_samelines(self, new.body, self.uncompressed_body, fmt)
|
||||
|
||||
def test_plain_response(self):
|
||||
rsp = Response(url="http://test.com", body=self.uncompressed_body)
|
||||
new = self.mw.process_response(None, rsp, self.spider)
|
||||
assert new is rsp
|
||||
assert_samelines(self, new.body, rsp.body)
|
||||
|
||||
def test_empty_response(self):
|
||||
rsp = Response(url="http://test.com", body=b"")
|
||||
new = self.mw.process_response(None, rsp, self.spider)
|
||||
assert new is rsp
|
||||
assert not rsp.body
|
||||
assert not new.body
|
||||
|
||||
def tearDown(self):
|
||||
del self.mw
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,10 +1,8 @@
|
|||
import unittest
|
||||
|
||||
import pytest
|
||||
from w3lib.http import basic_auth_header
|
||||
|
||||
from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Request
|
||||
from scrapy.spiders import Spider
|
||||
|
||||
|
|
@ -31,39 +29,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
|
|||
self.spider = TestSpiderLegacy("foo")
|
||||
|
||||
def test_auth(self):
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="Using HttpAuthMiddleware without http_auth_domain is deprecated",
|
||||
):
|
||||
with self.assertRaises(AttributeError):
|
||||
mw = HttpAuthMiddleware()
|
||||
mw.spider_opened(self.spider)
|
||||
|
||||
# initial request, sets the domain and sends the header
|
||||
req = Request("http://example.com/")
|
||||
assert mw.process_request(req, self.spider) is None
|
||||
self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
|
||||
|
||||
# subsequent request to the same domain, should send the header
|
||||
req = Request("http://example.com/")
|
||||
assert mw.process_request(req, self.spider) is None
|
||||
self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
|
||||
|
||||
# subsequent request to a different domain, shouldn't send the header
|
||||
req = Request("http://example-noauth.com/")
|
||||
assert mw.process_request(req, self.spider) is None
|
||||
self.assertNotIn("Authorization", req.headers)
|
||||
|
||||
def test_auth_already_set(self):
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="Using HttpAuthMiddleware without http_auth_domain is deprecated",
|
||||
):
|
||||
mw = HttpAuthMiddleware()
|
||||
mw.spider_opened(self.spider)
|
||||
req = Request("http://example.com/", headers=dict(Authorization="Digest 123"))
|
||||
assert mw.process_request(req, self.spider) is None
|
||||
self.assertEqual(req.headers["Authorization"], b"Digest 123")
|
||||
|
||||
|
||||
class HttpAuthMiddlewareTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ from gzip import GzipFile
|
|||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from unittest import SkipTest, TestCase
|
||||
from warnings import catch_warnings
|
||||
|
||||
from w3lib.encoding import resolve_encoding
|
||||
|
||||
|
|
@ -10,7 +9,7 @@ from scrapy.downloadermiddlewares.httpcompression import (
|
|||
ACCEPTED_ENCODINGS,
|
||||
HttpCompressionMiddleware,
|
||||
)
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import HtmlResponse, Request, Response
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.spiders import Spider
|
||||
|
|
@ -372,29 +371,3 @@ class HttpCompressionTest(TestCase):
|
|||
self.assertEqual(response.body, b"")
|
||||
self.assertStatsEqual("httpcompression/response_count", None)
|
||||
self.assertStatsEqual("httpcompression/response_bytes", None)
|
||||
|
||||
|
||||
class HttpCompressionSubclassTest(TestCase):
|
||||
def test_init_missing_stats(self):
|
||||
class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
crawler = get_crawler(Spider)
|
||||
with catch_warnings(record=True) as caught_warnings:
|
||||
HttpCompressionMiddlewareSubclass.from_crawler(crawler)
|
||||
messages = tuple(
|
||||
str(warning.message)
|
||||
for warning in caught_warnings
|
||||
if warning.category is ScrapyDeprecationWarning
|
||||
)
|
||||
self.assertEqual(
|
||||
messages,
|
||||
(
|
||||
(
|
||||
"HttpCompressionMiddleware subclasses must either modify "
|
||||
"their '__init__' method to support a 'stats' parameter "
|
||||
"or reimplement the 'from_crawler' method."
|
||||
),
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2835,200 +2835,6 @@ class FeedExportInitTest(unittest.TestCase):
|
|||
self.assertIsInstance(exporter, FeedExporter)
|
||||
|
||||
|
||||
class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage):
|
||||
def __init__(self, uri):
|
||||
super().__init__(uri)
|
||||
|
||||
|
||||
class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):
|
||||
"""Make sure that any feed exporter created by users before the
|
||||
introduction of the ``feed_options`` parameter continues to work as
|
||||
expected, and simply issues a warning."""
|
||||
|
||||
def test_init(self):
|
||||
settings_dict = {
|
||||
"FEED_URI": "file:///tmp/foobar",
|
||||
"FEED_STORAGES": {"file": StdoutFeedStorageWithoutFeedOptions},
|
||||
}
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
|
||||
):
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
|
||||
spider = scrapy.Spider("default")
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="StdoutFeedStorageWithoutFeedOptions does not support "
|
||||
"the 'feed_options' keyword argument.",
|
||||
):
|
||||
feed_exporter.open_spider(spider)
|
||||
|
||||
|
||||
class FileFeedStorageWithoutFeedOptions(FileFeedStorage):
|
||||
def __init__(self, uri):
|
||||
super().__init__(uri)
|
||||
|
||||
|
||||
class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
|
||||
"""Make sure that any feed exporter created by users before the
|
||||
introduction of the ``feed_options`` parameter continues to work as
|
||||
expected, and simply issues a warning."""
|
||||
|
||||
maxDiff = None
|
||||
|
||||
def test_init(self):
|
||||
with tempfile.NamedTemporaryFile() as temp:
|
||||
settings_dict = {
|
||||
"FEED_URI": f"file:///{temp.name}",
|
||||
"FEED_STORAGES": {"file": FileFeedStorageWithoutFeedOptions},
|
||||
}
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
|
||||
):
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider("default")
|
||||
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="FileFeedStorageWithoutFeedOptions does not support "
|
||||
"the 'feed_options' keyword argument.",
|
||||
):
|
||||
feed_exporter.open_spider(spider)
|
||||
|
||||
|
||||
class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
|
||||
def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs):
|
||||
super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs)
|
||||
|
||||
|
||||
class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, uri):
|
||||
return super().from_crawler(crawler, uri)
|
||||
|
||||
|
||||
class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
|
||||
"""Make sure that any feed exporter created by users before the
|
||||
introduction of the ``feed_options`` parameter continues to work as
|
||||
expected, and simply issues a warning."""
|
||||
|
||||
maxDiff = None
|
||||
|
||||
def setUp(self):
|
||||
skip_if_no_boto()
|
||||
|
||||
def test_init(self):
|
||||
settings_dict = {
|
||||
"FEED_URI": "file:///tmp/foobar",
|
||||
"FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptions},
|
||||
}
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
|
||||
):
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
|
||||
spider = scrapy.Spider("default")
|
||||
spider.crawler = crawler
|
||||
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="S3FeedStorageWithoutFeedOptions does not support "
|
||||
"the 'feed_options' keyword argument.",
|
||||
):
|
||||
feed_exporter.open_spider(spider)
|
||||
|
||||
def test_from_crawler(self):
|
||||
settings_dict = {
|
||||
"FEED_URI": "file:///tmp/foobar",
|
||||
"FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptionsWithFromCrawler},
|
||||
}
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
|
||||
):
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
|
||||
spider = scrapy.Spider("default")
|
||||
spider.crawler = crawler
|
||||
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
|
||||
"the 'feed_options' keyword argument.",
|
||||
):
|
||||
feed_exporter.open_spider(spider)
|
||||
|
||||
|
||||
class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):
|
||||
def __init__(self, uri, use_active_mode=False):
|
||||
super().__init__(uri)
|
||||
|
||||
|
||||
class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage):
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, uri):
|
||||
return super().from_crawler(crawler, uri)
|
||||
|
||||
|
||||
class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
|
||||
"""Make sure that any feed exporter created by users before the
|
||||
introduction of the ``feed_options`` parameter continues to work as
|
||||
expected, and simply issues a warning."""
|
||||
|
||||
maxDiff = None
|
||||
|
||||
def test_init(self):
|
||||
settings_dict = {
|
||||
"FEED_URI": "ftp://localhost/foo",
|
||||
"FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptions},
|
||||
}
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
|
||||
):
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
|
||||
spider = scrapy.Spider("default")
|
||||
spider.crawler = crawler
|
||||
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="FTPFeedStorageWithoutFeedOptions does not support "
|
||||
"the 'feed_options' keyword argument.",
|
||||
):
|
||||
feed_exporter.open_spider(spider)
|
||||
|
||||
def test_from_crawler(self):
|
||||
settings_dict = {
|
||||
"FEED_URI": "ftp://localhost/foo",
|
||||
"FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptionsWithFromCrawler},
|
||||
}
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
|
||||
):
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
|
||||
spider = scrapy.Spider("default")
|
||||
spider.crawler = crawler
|
||||
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
|
||||
"the 'feed_options' keyword argument.",
|
||||
):
|
||||
feed_exporter.open_spider(spider)
|
||||
|
||||
|
||||
class URIParamsTest:
|
||||
spider_name = "uri_params_spider"
|
||||
deprecated_options = False
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ Once we remove the references from scrapy, we can remove these tests.
|
|||
"""
|
||||
|
||||
import unittest
|
||||
import warnings
|
||||
from functools import partial
|
||||
|
||||
from itemloaders.processors import (
|
||||
|
|
@ -18,9 +17,6 @@ from itemloaders.processors import (
|
|||
|
||||
from scrapy.item import Field, Item
|
||||
from scrapy.loader import ItemLoader
|
||||
from scrapy.loader.common import wrap_loader_context
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.misc import extract_regex
|
||||
|
||||
|
||||
# test items
|
||||
|
|
@ -722,24 +718,5 @@ class FunctionProcessorTestCase(unittest.TestCase):
|
|||
self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]})
|
||||
|
||||
|
||||
class DeprecatedUtilityFunctionsTestCase(unittest.TestCase):
|
||||
def test_deprecated_wrap_loader_context(self):
|
||||
def function(*args):
|
||||
return None
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
wrap_loader_context(function, context={})
|
||||
|
||||
assert len(w) == 1
|
||||
assert issubclass(w[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
def test_deprecated_extract_regex(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
extract_regex(r"\w+", "this is a test")
|
||||
|
||||
assert len(w) == 1
|
||||
assert issubclass(w[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ from scrapy.pipelines.images import ImagesPipeline
|
|||
from scrapy.pipelines.media import MediaPipeline
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.signal import disconnect_all
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
|
@ -427,102 +426,6 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
|
|||
return super().image_downloaded(response, request, info)
|
||||
|
||||
|
||||
class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
|
||||
skip = skip_pillow
|
||||
|
||||
def setUp(self):
|
||||
settings_dict = {
|
||||
"IMAGES_STORE": "store-uri",
|
||||
"IMAGES_THUMBS": {"small": (50, 50)},
|
||||
}
|
||||
crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
|
||||
self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler)
|
||||
self.pipe.download_func = _mocked_download_func
|
||||
self.pipe.open_spider(None)
|
||||
self.item = dict(image_urls=["http://picsum.photos/id/1014/200/300"], images=[])
|
||||
|
||||
def _assert_method_called_with_warnings(self, method, message, warnings):
|
||||
self.assertIn(method, self.pipe._mockcalled)
|
||||
warningShown = False
|
||||
for warning in warnings:
|
||||
if (
|
||||
warning["message"] == message
|
||||
and warning["category"] == ScrapyDeprecationWarning
|
||||
):
|
||||
warningShown = True
|
||||
self.assertTrue(warningShown)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_media_to_download_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
"media_to_download(self, request, info) is deprecated, "
|
||||
"please use media_to_download(self, request, info, *, item=None)"
|
||||
)
|
||||
self._assert_method_called_with_warnings("media_to_download", message, warnings)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_media_downloaded_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
"media_downloaded(self, response, request, info) is deprecated, "
|
||||
"please use media_downloaded(self, response, request, info, *, item=None)"
|
||||
)
|
||||
self._assert_method_called_with_warnings("media_downloaded", message, warnings)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_file_downloaded_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
"file_downloaded(self, response, request, info) is deprecated, "
|
||||
"please use file_downloaded(self, response, request, info, *, item=None)"
|
||||
)
|
||||
self._assert_method_called_with_warnings("file_downloaded", message, warnings)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_file_path_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
"file_path(self, request, response=None, info=None) is deprecated, "
|
||||
"please use file_path(self, request, response=None, info=None, *, item=None)"
|
||||
)
|
||||
self._assert_method_called_with_warnings("file_path", message, warnings)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_thumb_path_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
"thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, "
|
||||
"please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)"
|
||||
)
|
||||
self._assert_method_called_with_warnings("thumb_path", message, warnings)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_get_images_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
"get_images(self, response, request, info) is deprecated, "
|
||||
"please use get_images(self, response, request, info, *, item=None)"
|
||||
)
|
||||
self._assert_method_called_with_warnings("get_images", message, warnings)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_image_downloaded_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
"image_downloaded(self, response, request, info) is deprecated, "
|
||||
"please use image_downloaded(self, response, request, info, *, item=None)"
|
||||
)
|
||||
self._assert_method_called_with_warnings("image_downloaded", message, warnings)
|
||||
|
||||
|
||||
class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):
|
||||
def _assert_request_no3xx(self, pipeline_class, settings):
|
||||
pipe = pipeline_class(settings=Settings(settings))
|
||||
|
|
|
|||
Loading…
Reference in New Issue