RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Deprecation removals. (#6500) 

* Deprecation removals.

* Clean up the default pytest filterwarnings.

* Remove test_get_images_old().

* Redo boto-requiring test filtering.

* Remove an unused function.

* Improve the Crawler.crawl() error message.

* Fix the test.
This commit is contained in:
Andrey Rakhmatullin 2024-10-31 18:06:22 +05:00 committed by GitHub
parent d2bdbad8c8
commit d85c39f5bc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 103 additions and 587 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
conftest.py
View File

@ -89,6 +89,30 @@ def requires_uvloop(request):
pytest.skip("uvloop is not installed")
@pytest.fixture(autouse=True)
def requires_botocore(request):
if not request.node.get_closest_marker("requires_botocore"):
return
try:
import botocore
del botocore
except ImportError:
pytest.skip("botocore is not installed")
@pytest.fixture(autouse=True)
def requires_boto3(request):
if not request.node.get_closest_marker("requires_boto3"):
return
try:
import boto3
del boto3
except ImportError:
pytest.skip("boto3 is not installed")
def pytest_configure(config):
if config.getoption("--reactor") == "asyncio":
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")

5
docs/topics/commands.rst
View File

@ -278,8 +278,6 @@ Supported options:
* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``)
* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O``
Usage examples::
$ scrapy crawl myspider
@ -291,9 +289,6 @@ Usage examples::
$ scrapy crawl -O myfile:json myspider
[ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ]
$ scrapy crawl -o myfile -t csv myspider
[ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
.. command:: check
check

2
extras/scrapy_zsh_completion
View File

@ -41,7 +41,6 @@ _scrapy() {
(runspider)
local options=(
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider file:_files -g \*.py'
)
@ -99,7 +98,6 @@ _scrapy() {
(crawl)
local options=(
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider:_scrapy_spiders'
)

6
pytest.ini
View File

@ -21,8 +21,6 @@ markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
requires_uvloop: marks tests as only enabled when uvloop is known to be working
requires_botocore: marks tests that need botocore (but not boto3)
requires_boto3: marks tests that need botocore and boto3
filterwarnings =
ignore:scrapy.downloadermiddlewares.decompression is deprecated
ignore:Module scrapy.utils.reqser is deprecated
ignore:typing.re is deprecated
ignore:typing.io is deprecated

9
scrapy/commands/__init__.py
View File

@ -162,12 +162,6 @@ class BaseRunSpiderCommand(ScrapyCommand):
help="dump scraped items into FILE, overwriting any existing file,"
" to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)",
)
parser.add_argument(
"-t",
"--output-format",
metavar="FORMAT",
help="format to use for dumping items",
)
def process_options(self, args: list[str], opts: argparse.Namespace) -> None:
super().process_options(args, opts)
@ -179,8 +173,7 @@ class BaseRunSpiderCommand(ScrapyCommand):
feeds = feed_process_params_from_cli(
self.settings,
opts.output,
opts.output_format,
opts.overwrite_output,
overwrite_output=opts.overwrite_output,
)
self.settings.set("FEEDS", feeds, priority="cmdline")

8
scrapy/crawler.py
View File

@ -3,7 +3,6 @@ from __future__ import annotations
import logging
import pprint
import signal
import warnings
from typing import TYPE_CHECKING, Any, TypeVar, cast
from twisted.internet.defer import (
@ -17,7 +16,6 @@ from zope.interface.verify import verifyClass
from scrapy import Spider, signals
from scrapy.addons import AddonManager
from scrapy.core.engine import ExecutionEngine
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager
from scrapy.interfaces import ISpiderLoader
from scrapy.logformatter import LogFormatter
@ -142,10 +140,8 @@ class Crawler:
if self.crawling:
raise RuntimeError("Crawling already taking place")
if self._started:
warnings.warn(
"Running Crawler.crawl() more than once is deprecated.",
ScrapyDeprecationWarning,
stacklevel=2,
raise RuntimeError(
"Cannot run Crawler.crawl() more than once on the same instance."
)
self.crawling = self._started = True

18
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -1,6 +1,5 @@
from __future__ import annotations
import warnings
from itertools import chain
from logging import getLogger
from typing import TYPE_CHECKING, Any
@ -15,7 +14,6 @@ from scrapy.utils._compression import (
_unbrotli,
_unzstd,
)
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.gz import gunzip
if TYPE_CHECKING:
@ -72,21 +70,7 @@ class HttpCompressionMiddleware:
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("COMPRESSION_ENABLED"):
raise NotConfigured
try:
return cls(crawler=crawler)
except TypeError:
warnings.warn(
"HttpCompressionMiddleware subclasses must either modify "
"their '__init__' method to support a 'crawler' parameter or "
"reimplement their 'from_crawler' method.",
ScrapyDeprecationWarning,
)
mw = cls()
mw.stats = crawler.stats
mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
crawler.signals.connect(mw.open_spider, signals.spider_opened)
return mw
return cls(crawler=crawler)
def open_spider(self, spider: Spider) -> None:
if hasattr(spider, "download_maxsize"):

45
scrapy/downloadermiddlewares/retry.py
View File

@ -12,12 +12,10 @@ once the spider has finished crawling all regular (non-failed) pages.
from __future__ import annotations
import warnings
from logging import Logger, getLogger
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.settings import BaseSettings, Settings
from scrapy.exceptions import NotConfigured
from scrapy.utils.misc import load_object
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
@ -29,33 +27,13 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.http.request import Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
retry_logger = getLogger(__name__)
def backwards_compatibility_getattr(self: Any, name: str) -> tuple[Any, ...]:
if name == "EXCEPTIONS_TO_RETRY":
warnings.warn(
"Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
"Use the RETRY_EXCEPTIONS setting instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return tuple(
load_object(x) if isinstance(x, str) else x
for x in Settings().getlist("RETRY_EXCEPTIONS")
)
raise AttributeError(
f"{self.__class__.__name__!r} object has no attribute {name!r}"
)
class BackwardsCompatibilityMetaclass(type):
__getattr__ = backwards_compatibility_getattr
def get_retry_request(
request: Request,
*,
@ -144,22 +122,17 @@ def get_retry_request(
return None
class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
class RetryMiddleware:
def __init__(self, settings: BaseSettings):
if not settings.getbool("RETRY_ENABLED"):
raise NotConfigured
self.max_retry_times = settings.getint("RETRY_TIMES")
self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}
self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
try:
self.exceptions_to_retry = self.__getattribute__("EXCEPTIONS_TO_RETRY")
except AttributeError:
# If EXCEPTIONS_TO_RETRY is not "overridden"
self.exceptions_to_retry = tuple(
load_object(x) if isinstance(x, str) else x
for x in settings.getlist("RETRY_EXCEPTIONS")
)
self.exceptions_to_retry = tuple(
load_object(x) if isinstance(x, str) else x
for x in settings.getlist("RETRY_EXCEPTIONS")
)
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
@ -199,5 +172,3 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
max_retry_times=max_retry_times,
priority_adjust=priority_adjust,
)
__getattr__ = backwards_compatibility_getattr

80
scrapy/extensions/feedexport.py
View File

@ -26,10 +26,8 @@ from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.extensions.postprocessing import PostProcessingManager
from scrapy.settings import Settings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import build_from_crawler, load_object
@ -48,13 +46,6 @@ if TYPE_CHECKING:
from scrapy.exporters import BaseItemExporter
from scrapy.settings import BaseSettings
try:
import boto3 # noqa: F401
IS_BOTO3_AVAILABLE = True
except ImportError:
IS_BOTO3_AVAILABLE = False
logger = logging.getLogger(__name__)
@ -217,8 +208,10 @@ class S3FeedStorage(BlockingFeedStorage):
session_token: str | None = None,
region_name: str | None = None,
):
if not is_botocore_available():
raise NotConfigured("missing botocore library")
try:
import boto3.session
except ImportError:
raise NotConfigured("missing boto3 library")
u = urlparse(uri)
assert u.hostname
self.bucketname: str = u.hostname
@ -229,42 +222,16 @@ class S3FeedStorage(BlockingFeedStorage):
self.acl: str | None = acl
self.endpoint_url: str | None = endpoint_url
self.region_name: str | None = region_name
# It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client,
# there seems to be no good way to infer it statically.
self.s3_client: Any
if IS_BOTO3_AVAILABLE:
import boto3.session
boto3_session = boto3.session.Session()
self.s3_client = boto3_session.client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
region_name=self.region_name,
)
else:
warnings.warn(
"`botocore` usage has been deprecated for S3 feed "
"export, please use `boto3` to avoid problems",
category=ScrapyDeprecationWarning,
)
import botocore.session
botocore_session = botocore.session.get_session()
self.s3_client = botocore_session.create_client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
region_name=self.region_name,
)
boto3_session = boto3.session.Session()
self.s3_client = boto3_session.client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
region_name=self.region_name,
)
if feed_options and feed_options.get("overwrite", True) is False:
logger.warning(
@ -295,17 +262,10 @@ class S3FeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file: IO[bytes]) -> None:
file.seek(0)
kwargs: dict[str, Any]
if IS_BOTO3_AVAILABLE:
kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
self.s3_client.upload_fileobj(
Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
)
else:
kwargs = {"ACL": self.acl} if self.acl else {}
self.s3_client.put_object(
Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
)
kwargs: dict[str, Any] = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
self.s3_client.upload_fileobj(
Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
)
file.close()
@ -464,12 +424,6 @@ class FeedSlot:
self._exporting = False
_FeedSlot = create_deprecated_class(
name="_FeedSlot",
new_class=FeedSlot,
)
class FeedExporter:
_pending_deferreds: list[Deferred[None]] = []

57
scrapy/pipelines/images.py
View File

@ -8,14 +8,13 @@ from __future__ import annotations
import functools
import hashlib
import warnings
from contextlib import suppress
from io import BytesIO
from typing import TYPE_CHECKING, Any, cast
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import (
@ -27,7 +26,7 @@ from scrapy.pipelines.files import (
_md5sum,
)
from scrapy.settings import Settings
from scrapy.utils.python import get_func_args, to_bytes
from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
@ -42,18 +41,6 @@ if TYPE_CHECKING:
from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
class NoimagesDrop(DropItem):
"""Product with no images exception"""
def __init__(self, *args: Any, **kwargs: Any):
warnings.warn(
"The NoimagesDrop class is deprecated",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
super().__init__(*args, **kwargs)
class ImageException(FileException):
"""General image error exception"""
@ -120,8 +107,6 @@ class ImagesPipeline(FilesPipeline):
resolve("IMAGES_THUMBS"), self.THUMBS
)
self._deprecated_convert_image: bool | None = None
@classmethod
def from_settings(cls, settings: Settings) -> Self:
s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"])
@ -203,49 +188,25 @@ class ImagesPipeline(FilesPipeline):
f"{self.min_width}x{self.min_height})"
)
if self._deprecated_convert_image is None:
self._deprecated_convert_image = "response_body" not in get_func_args(
self.convert_image
)
if self._deprecated_convert_image:
warnings.warn(
f"{self.__class__.__name__}.convert_image() method overridden in a deprecated way, "
"overridden method does not accept response_body argument.",
category=ScrapyDeprecationWarning,
)
if self._deprecated_convert_image:
image, buf = self.convert_image(orig_image)
else:
image, buf = self.convert_image(
orig_image, response_body=BytesIO(response.body)
)
image, buf = self.convert_image(
orig_image, response_body=BytesIO(response.body)
)
yield path, image, buf
for thumb_id, size in self.thumbs.items():
thumb_path = self.thumb_path(
request, thumb_id, response=response, info=info, item=item
)
if self._deprecated_convert_image:
thumb_image, thumb_buf = self.convert_image(image, size)
else:
thumb_image, thumb_buf = self.convert_image(image, size, buf)
thumb_image, thumb_buf = self.convert_image(image, size, response_body=buf)
yield thumb_path, thumb_image, thumb_buf
def convert_image(
self,
image: Image.Image,
size: tuple[int, int] | None = None,
response_body: BytesIO | None = None,
*,
response_body: BytesIO,
) -> tuple[Image.Image, BytesIO]:
if response_body is None:
warnings.warn(
f"{self.__class__.__name__}.convert_image() method called in a deprecated way, "
"method called without response_body argument.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if image.format in ("PNG", "WEBP") and image.mode == "RGBA":
background = self._Image.new("RGBA", image.size, (255, 255, 255))
background.paste(image, image)
@ -268,7 +229,7 @@ class ImagesPipeline(FilesPipeline):
except AttributeError:
resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined]
image.thumbnail(size, resampling_filter)
elif response_body is not None and image.format == "JPEG":
elif image.format == "JPEG":
return image, response_body
buf = BytesIO()

48
scrapy/utils/conf.py
View File

@ -3,14 +3,13 @@ from __future__ import annotations
import numbers
import os
import sys
import warnings
from collections.abc import Iterable
from configparser import ConfigParser
from operator import itemgetter
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, cast
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
from scrapy.exceptions import UsageError
from scrapy.settings import BaseSettings
from scrapy.utils.deprecate import update_classpath
from scrapy.utils.python import without_none_values
@ -21,7 +20,7 @@ if TYPE_CHECKING:
def build_component_list(
compdict: MutableMapping[Any, Any],
custom: Any = None,
*,
convert: Callable[[Any], Any] = update_classpath,
) -> list[Any]:
"""Compose a component list from a { class: order } dictionary."""
@ -60,19 +59,6 @@ def build_component_list(
"please provide a real number or None instead"
)
if custom is not None:
warnings.warn(
"The 'custom' attribute of build_component_list() is deprecated. "
"Please merge its value into 'compdict' manually or change your "
"code to use Settings.getwithbase().",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if isinstance(custom, (list, tuple)):
_check_components(custom)
return type(custom)(convert(c) for c in custom) # type: ignore[return-value]
compdict.update(custom)
_validate_values(compdict)
compdict = without_none_values(_map_keys(compdict))
return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
@ -159,7 +145,7 @@ def feed_complete_default_values_from_settings(
def feed_process_params_from_cli(
settings: BaseSettings,
output: list[str],
output_format: str | None = None,
*,
overwrite_output: list[str] | None = None,
) -> dict[str, dict[str, Any]]:
"""
@ -186,37 +172,9 @@ def feed_process_params_from_cli(
raise UsageError(
"Please use only one of -o/--output and -O/--overwrite-output"
)
if output_format:
raise UsageError(
"-t/--output-format is a deprecated command line option"
" and does not work in combination with -O/--overwrite-output."
" To specify a format please specify it after a colon at the end of the"
" output URI (i.e. -O <URI>:<FORMAT>)."
" Example working in the tutorial: "
"scrapy crawl quotes -O quotes.json:json"
)
output = overwrite_output
overwrite = True
if output_format:
if len(output) == 1:
check_valid_format(output_format)
message = (
"The -t/--output-format command line option is deprecated in favor of "
"specifying the output format within the output URI using the -o/--output or the"
" -O/--overwrite-output option (i.e. -o/-O <URI>:<FORMAT>). See the documentation"
" of the -o or -O option or the following examples for more information. "
"Examples working in the tutorial: "
"scrapy crawl quotes -o quotes.csv:csv or "
"scrapy crawl quotes -O quotes.json:json"
)
warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
return {output[0]: {"format": output_format}}
raise UsageError(
"The -t command-line option cannot be used if multiple output "
"URIs are specified"
)
result: dict[str, dict[str, Any]] = {}
for element in output:
try:

19
scrapy/utils/reactor.py
View File

@ -4,12 +4,11 @@ import asyncio
import sys
from contextlib import suppress
from typing import TYPE_CHECKING, Any, Generic, TypeVar
from warnings import catch_warnings, filterwarnings, warn
from warnings import catch_warnings, filterwarnings
from twisted.internet import asyncioreactor, error
from twisted.internet.base import DelayedCall
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.misc import load_object
if TYPE_CHECKING:
@ -79,22 +78,6 @@ def set_asyncio_event_loop_policy() -> None:
_get_asyncio_event_loop_policy()
def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:
warn(
"Call to deprecated function "
"scrapy.utils.reactor.get_asyncio_event_loop_policy().\n"
"\n"
"Please use get_event_loop, new_event_loop and set_event_loop"
" from asyncio instead, as the corresponding policy methods may lead"
" to unexpected behaviour.\n"
"This function is replaced by set_asyncio_event_loop_policy and"
" is meant to be used only when the reactor is being installed.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return _get_asyncio_event_loop_policy()
def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:
policy = asyncio.get_event_loop_policy()
if sys.platform == "win32" and not isinstance(

10
scrapy/utils/request.py
View File

@ -30,17 +30,9 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]:
for header in headers:
if header in request.headers:
yield header
yield from request.headers.getlist(header)
_fingerprint_cache: WeakKeyDictionary[
Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]
]
_fingerprint_cache = WeakKeyDictionary()
] = WeakKeyDictionary()
def fingerprint(

8
tests/test_crawler.py
View File

@ -8,7 +8,6 @@ import warnings
from pathlib import Path
from typing import Any
import pytest
from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn
from pytest import mark, raises
@ -82,13 +81,10 @@ class CrawlerTestCase(BaseCrawlerTest):
Crawler(DefaultSpider())
@inlineCallbacks
def test_crawler_crawl_twice_deprecated(self):
def test_crawler_crawl_twice_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
yield crawler.crawl()
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Running Crawler.crawl\(\) more than once is deprecated",
):
with raises(RuntimeError, match="more than once on the same instance"):
yield crawler.crawl()
def test_get_addon(self):

7
tests/test_downloader_handlers.py
View File

@ -8,6 +8,7 @@ from pathlib import Path
from tempfile import mkdtemp, mkstemp
from unittest import SkipTest, mock
import pytest
from testfixtures import LogCapture
from twisted.cred import checkers, credentials, portal
from twisted.internet import defer, error, reactor
@ -32,7 +33,7 @@ from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler, skip_if_no_boto
from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver import (
Echo,
@ -824,9 +825,9 @@ class HttpDownloadHandlerMock:
return request
@pytest.mark.requires_botocore
class S3AnonTestCase(unittest.TestCase):
def setUp(self):
skip_if_no_boto()
crawler = get_crawler()
self.s3reqh = build_from_crawler(
S3DownloadHandler,
@ -845,6 +846,7 @@ class S3AnonTestCase(unittest.TestCase):
self.assertEqual(httpreq.url, "http://aws-publicdatasets.s3.amazonaws.com/")
@pytest.mark.requires_botocore
class S3TestCase(unittest.TestCase):
download_handler_cls: type = S3DownloadHandler
@ -856,7 +858,6 @@ class S3TestCase(unittest.TestCase):
AWS_SECRET_ACCESS_KEY = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o"
def setUp(self):
skip_if_no_boto()
crawler = get_crawler()
s3reqh = build_from_crawler(
S3DownloadHandler,

29
tests/test_downloadermiddleware_httpcompression.py
View File

@ -3,7 +3,6 @@ from io import BytesIO
from logging import WARNING
from pathlib import Path
from unittest import SkipTest, TestCase
from warnings import catch_warnings
from testfixtures import LogCapture
from w3lib.encoding import resolve_encoding
@ -12,7 +11,7 @@ from scrapy.downloadermiddlewares.httpcompression import (
ACCEPTED_ENCODINGS,
HttpCompressionMiddleware,
)
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
@ -700,29 +699,3 @@ class HttpCompressionTest(TestCase):
except ImportError:
raise SkipTest("no zstd support (zstandard)")
self._test_download_warnsize_request_meta("zstd")
class HttpCompressionSubclassTest(TestCase):
def test_init_missing_stats(self):
class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):
def __init__(self):
super().__init__()
crawler = get_crawler(Spider)
with catch_warnings(record=True) as caught_warnings:
HttpCompressionMiddlewareSubclass.from_crawler(crawler)
messages = tuple(
str(warning.message)
for warning in caught_warnings
if warning.category is ScrapyDeprecationWarning
)
self.assertEqual(
messages,
(
(
"HttpCompressionMiddleware subclasses must either modify "
"their '__init__' method to support a 'crawler' parameter "
"or reimplement their 'from_crawler' method."
),
),
)

32
tests/test_downloadermiddleware_retry.py
View File

@ -1,6 +1,5 @@
import logging
import unittest
import warnings
from testfixtures import LogCapture
from twisted.internet import defer
@ -122,37 +121,6 @@ class RetryTest(unittest.TestCase):
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
self._test_retry_exception(req, exc("foo"), mw)
def test_exception_to_retry_custom_middleware(self):
exc = ValueError
with warnings.catch_warnings(record=True) as warns:
class MyRetryMiddleware(RetryMiddleware):
EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,)
self.assertEqual(len(warns), 1)
mw2 = MyRetryMiddleware.from_crawler(self.crawler)
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
req = mw2.process_exception(req, exc("foo"), self.spider)
assert isinstance(req, Request)
self.assertEqual(req.meta["retry_times"], 1)
def test_exception_to_retry_custom_middleware_self(self):
class MyRetryMiddleware(RetryMiddleware):
def process_exception(self, request, exception, spider):
if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
return self._retry(request, exception, spider)
exc = OSError
mw2 = MyRetryMiddleware.from_crawler(self.crawler)
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
with warnings.catch_warnings(record=True) as warns:
req = mw2.process_exception(req, exc("foo"), self.spider)
assert isinstance(req, Request)
self.assertEqual(req.meta["retry_times"], 1)
self.assertEqual(len(warns), 1)
def _test_retry_exception(self, req, exception, mw=None):
if mw is None:
mw = self.mw

65
tests/test_feedexport.py
View File

@ -37,7 +37,6 @@ from scrapy import signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exporters import CsvItemExporter, JsonItemExporter
from scrapy.extensions.feedexport import (
IS_BOTO3_AVAILABLE,
BlockingFeedStorage,
FeedExporter,
FeedSlot,
@ -50,7 +49,7 @@ from scrapy.extensions.feedexport import (
)
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_boto
from scrapy.utils.test import get_crawler, mock_google_cloud_storage
from tests.mockserver import MockFTPServer, MockServer
from tests.spiders import ItemSpider
@ -240,10 +239,8 @@ class BlockingFeedStorageTest(unittest.TestCase):
self.assertRaises(OSError, b.open, spider=spider)
@pytest.mark.requires_boto3
class S3FeedStorageTest(unittest.TestCase):
def setUp(self):
skip_if_no_boto()
def test_parse_credentials(self):
aws_credentials = {
"AWS_ACCESS_KEY_ID": "settings_key",
@ -292,38 +289,12 @@ class S3FeedStorageTest(unittest.TestCase):
file = mock.MagicMock()
if IS_BOTO3_AVAILABLE:
storage.s3_client = mock.MagicMock()
yield storage.store(file)
self.assertEqual(
storage.s3_client.upload_fileobj.call_args,
mock.call(Bucket=bucket, Key=key, Fileobj=file),
)
else:
from botocore.stub import Stubber
with Stubber(storage.s3_client) as stub:
stub.add_response(
"put_object",
expected_params={
"Body": file,
"Bucket": bucket,
"Key": key,
},
service_response={},
)
yield storage.store(file)
stub.assert_no_pending_responses()
self.assertEqual(
file.method_calls,
[
mock.call.seek(0),
# The call to read does not happen with Stubber
mock.call.close(),
],
)
storage.s3_client = mock.MagicMock()
yield storage.store(file)
self.assertEqual(
storage.s3_client.upload_fileobj.call_args,
mock.call(Bucket=bucket, Key=key, Fileobj=file),
)
def test_init_without_acl(self):
storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")
@ -459,14 +430,11 @@ class S3FeedStorageTest(unittest.TestCase):
storage.s3_client = mock.MagicMock()
yield storage.store(BytesIO(b"test file"))
if IS_BOTO3_AVAILABLE:
acl = (
storage.s3_client.upload_fileobj.call_args[1]
.get("ExtraArgs", {})
.get("ACL")
)
else:
acl = storage.s3_client.put_object.call_args[1].get("ACL")
acl = (
storage.s3_client.upload_fileobj.call_args[1]
.get("ExtraArgs", {})
.get("ACL")
)
self.assertIsNone(acl)
@defer.inlineCallbacks
@ -480,10 +448,7 @@ class S3FeedStorageTest(unittest.TestCase):
storage.s3_client = mock.MagicMock()
yield storage.store(BytesIO(b"test file"))
if IS_BOTO3_AVAILABLE:
acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
else:
acl = storage.s3_client.put_object.call_args[1]["ACL"]
acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
self.assertEqual(acl, "custom-acl")
def test_overwrite_default(self):
@ -2647,9 +2612,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12
)
@pytest.mark.requires_boto3
@defer.inlineCallbacks
def test_s3_export(self):
skip_if_no_boto()
bucket = "mybucket"
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),

7
tests/test_pipeline_files.py
View File

@ -11,6 +11,7 @@ from unittest import mock
from urllib.parse import urlparse
import attr
import pytest
from itemadapter import ItemAdapter
from twisted.internet import defer
from twisted.trial import unittest
@ -30,7 +31,6 @@ from scrapy.utils.test import (
get_crawler,
get_ftp_content_and_delete,
get_gcs_content_and_delete,
skip_if_no_boto,
)
from tests.mockserver import MockFTPServer
@ -507,11 +507,10 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
self.assertEqual(fs_store.basedir, str(path))
@pytest.mark.requires_botocore
class TestS3FilesStore(unittest.TestCase):
@defer.inlineCallbacks
def test_persist(self):
skip_if_no_boto()
bucket = "mybucket"
key = "export.csv"
uri = f"s3://{bucket}/{key}"
@ -557,8 +556,6 @@ class TestS3FilesStore(unittest.TestCase):
@defer.inlineCallbacks
def test_stat(self):
skip_if_no_boto()
bucket = "mybucket"
key = "export.csv"
uri = f"s3://{bucket}/{key}"

135
tests/test_pipeline_images.py
View File

@ -1,24 +1,19 @@
from __future__ import annotations
import dataclasses
import hashlib
import io
import random
import warnings
from shutil import rmtree
from tempfile import mkdtemp
from unittest.mock import patch
import attr
from itemadapter import ItemAdapter
from twisted.trial import unittest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop
from scrapy.pipelines.images import ImageException, ImagesPipeline
from scrapy.settings import Settings
from scrapy.utils.python import to_bytes
skip_pillow: str | None
try:
@ -159,7 +154,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
with self.assertRaises(ImageException):
next(self.pipeline.get_images(response=resp3, request=req, info=object()))
def test_get_images_new(self):
def test_get_images(self):
self.pipeline.min_width = 0
self.pipeline.min_height = 0
self.pipeline.thumbs = {"small": (20, 20)}
@ -185,101 +180,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
self.assertEqual(thumb_img, thumb_img)
self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue())
def test_get_images_old(self):
self.pipeline.thumbs = {"small": (20, 20)}
orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue())
req = Request(url="https://dev.mydeco.com/mydeco.gif")
def overridden_convert_image(image, size=None):
im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
return im, buf
with patch.object(self.pipeline, "convert_image", overridden_convert_image):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
get_images_gen = self.pipeline.get_images(
response=resp, request=req, info=object()
)
path, new_im, new_buf = next(get_images_gen)
self.assertEqual(
path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
)
self.assertEqual(orig_im.mode, new_im.mode)
self.assertEqual(orig_im.getcolors(), new_im.getcolors())
self.assertEqual(buf.getvalue(), new_buf.getvalue())
thumb_path, thumb_img, thumb_buf = next(get_images_gen)
self.assertEqual(
thumb_path,
"thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg",
)
self.assertEqual(orig_im.mode, thumb_img.mode)
self.assertEqual(orig_im.getcolors(), thumb_img.getcolors())
self.assertEqual(buf.getvalue(), thumb_buf.getvalue())
expected_warning_msg = (
".convert_image() method overridden in a deprecated way, "
"overridden method does not accept response_body argument."
)
self.assertEqual(
len(
[
warning
for warning in w
if expected_warning_msg in str(warning.message)
]
),
1,
)
def test_convert_image_old(self):
# tests for old API
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
SIZE = (100, 100)
# straight forward case: RGB and JPEG
COLOUR = (0, 127, 255)
im, _ = _create_image("JPEG", "RGB", SIZE, COLOUR)
converted, _ = self.pipeline.convert_image(im)
self.assertEqual(converted.mode, "RGB")
self.assertEqual(converted.getcolors(), [(10000, COLOUR)])
# check that thumbnail keep image ratio
thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25))
self.assertEqual(thumbnail.mode, "RGB")
self.assertEqual(thumbnail.size, (10, 10))
# transparency case: RGBA and PNG
COLOUR = (0, 127, 255, 50)
im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR)
converted, _ = self.pipeline.convert_image(im)
self.assertEqual(converted.mode, "RGB")
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
# transparency case with palette: P and PNG
COLOUR = (0, 127, 255, 50)
im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR)
im = im.convert("P")
converted, _ = self.pipeline.convert_image(im)
self.assertEqual(converted.mode, "RGB")
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
# ensure that we received deprecation warnings
expected_warning_msg = ".convert_image() method called in a deprecated way"
self.assertTrue(
len(
[
warning
for warning in w
if expected_warning_msg in str(warning.message)
]
)
== 4
)
def test_convert_image_new(self):
# tests for new API
def test_convert_image(self):
SIZE = (100, 100)
# straight forward case: RGB and JPEG
COLOUR = (0, 127, 255)
@ -313,19 +214,6 @@ class ImagesPipelineTestCase(unittest.TestCase):
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
class DeprecatedImagesPipeline(ImagesPipeline):
def file_key(self, url):
return self.image_key(url)
def image_key(self, url):
image_guid = hashlib.sha1(to_bytes(url)).hexdigest()
return f"empty/{image_guid}.jpg"
def thumb_key(self, url, thumb_id):
thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest()
return f"thumbsup/{thumb_id}/{thumb_guid}.jpg"
class ImagesPipelineTestCaseFieldsMixin:
skip = skip_pillow
@ -627,23 +515,6 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value)
class NoimagesDropTestCase(unittest.TestCase):
def test_deprecation_warning(self):
arg = ""
with warnings.catch_warnings(record=True) as w:
NoimagesDrop(arg)
self.assertEqual(len(w), 1)
self.assertEqual(w[0].category, ScrapyDeprecationWarning)
with warnings.catch_warnings(record=True) as w:
class SubclassedNoimagesDrop(NoimagesDrop):
pass
SubclassedNoimagesDrop(arg)
self.assertEqual(len(w), 1)
self.assertEqual(w[0].category, ScrapyDeprecationWarning)
def _create_image(format, *a, **kw):
buf = io.BytesIO()
Image.new(*a, **kw).save(buf, format)

72
tests/test_utils_conf.py
View File

@ -1,9 +1,6 @@
import unittest
import warnings
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
from scrapy.exceptions import UsageError
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.conf import (
arglist_to_dict,
@ -20,50 +17,6 @@ class BuildComponentListTest(unittest.TestCase):
build_component_list(d, convert=lambda x: x), ["one", "four", "three"]
)
def test_backward_compatible_build_dict(self):
base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None}
custom = {"two": None, "three": 8, "four": 4}
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
self.assertEqual(
build_component_list(base, custom, convert=lambda x: x),
["one", "four", "five", "three"],
)
def test_return_list(self):
custom = ["a", "b", "c"]
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
self.assertEqual(
build_component_list(None, custom, convert=lambda x: x), custom
)
def test_map_dict(self):
custom = {"one": 1, "two": 2, "three": 3}
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
self.assertEqual(
build_component_list({}, custom, convert=lambda x: x.upper()),
["ONE", "TWO", "THREE"],
)
def test_map_list(self):
custom = ["a", "b", "c"]
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
self.assertEqual(
build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"]
)
def test_duplicate_components_in_dict(self):
duplicate_dict = {"one": 1, "two": 2, "ONE": 4}
with self.assertRaises(ValueError):
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
build_component_list({}, duplicate_dict, convert=lambda x: x.lower())
def test_duplicate_components_in_list(self):
duplicate_list = ["a", "b", "a"]
with self.assertRaises(ValueError) as cm:
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
build_component_list(None, duplicate_list, convert=lambda x: x)
self.assertIn(str(duplicate_list), str(cm.exception))
def test_duplicate_components_in_basesettings(self):
# Higher priority takes precedence
duplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0)
@ -92,11 +45,6 @@ class BuildComponentListTest(unittest.TestCase):
"c": 22222222222222222222,
}
self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"])
# raise exception for invalid values
d = {"one": "5"}
with self.assertRaises(ValueError):
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
build_component_list({}, d, convert=lambda x: x)
class UtilsConfTestCase(unittest.TestCase):
@ -115,7 +63,6 @@ class FeedExportConfigTestCase(unittest.TestCase):
feed_process_params_from_cli,
settings,
["items.dat"],
"noformat",
)
def test_feed_export_config_mismatch(self):
@ -125,18 +72,8 @@ class FeedExportConfigTestCase(unittest.TestCase):
feed_process_params_from_cli,
settings,
["items1.dat", "items2.dat"],
"noformat",
)
def test_feed_export_config_backward_compatible(self):
with warnings.catch_warnings(record=True) as cw:
settings = Settings()
self.assertEqual(
{"items.dat": {"format": "csv"}},
feed_process_params_from_cli(settings, ["items.dat"], "csv"),
)
self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
def test_feed_export_config_explicit_formats(self):
settings = Settings()
self.assertEqual(
@ -174,7 +111,9 @@ class FeedExportConfigTestCase(unittest.TestCase):
settings = Settings()
self.assertEqual(
{"output.json": {"format": "json", "overwrite": True}},
feed_process_params_from_cli(settings, [], None, ["output.json"]),
feed_process_params_from_cli(
settings, [], overwrite_output=["output.json"]
),
)
def test_output_and_overwrite_output(self):
@ -182,8 +121,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
feed_process_params_from_cli(
Settings(),
["output1.json"],
None,
["output2.json"],
overwrite_output=["output2.json"],
)
def test_feed_complete_default_values_from_settings_empty(self):

4
tox.ini
View File

@ -241,7 +241,7 @@ deps =
{[testenv]deps}
botocore>=1.4.87
commands =
pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore}
[testenv:botocore-pinned]
basepython = {[pinned]basepython}
@ -252,4 +252,4 @@ install_command = {[pinned]install_command}
setenv =
{[pinned]setenv}
commands =
pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore}