From d85c39f5bcd728915fccece86f2b2e4ef37c0e53 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 31 Oct 2024 18:06:22 +0500 Subject: [PATCH] Deprecation removals. (#6500) * Deprecation removals. * Clean up the default pytest filterwarnings. * Remove test_get_images_old(). * Redo boto-requiring test filtering. * Remove an unused function. * Improve the Crawler.crawl() error message. * Fix the test. --- conftest.py | 24 ++++ docs/topics/commands.rst | 5 - extras/scrapy_zsh_completion | 2 - pytest.ini | 6 +- scrapy/commands/__init__.py | 9 +- scrapy/crawler.py | 8 +- .../downloadermiddlewares/httpcompression.py | 18 +-- scrapy/downloadermiddlewares/retry.py | 45 ++---- scrapy/extensions/feedexport.py | 80 +++-------- scrapy/pipelines/images.py | 57 ++------ scrapy/utils/conf.py | 48 +------ scrapy/utils/reactor.py | 19 +-- scrapy/utils/request.py | 10 +- tests/test_crawler.py | 8 +- tests/test_downloader_handlers.py | 7 +- ...st_downloadermiddleware_httpcompression.py | 29 +--- tests/test_downloadermiddleware_retry.py | 32 ----- tests/test_feedexport.py | 65 ++------- tests/test_pipeline_files.py | 7 +- tests/test_pipeline_images.py | 135 +----------------- tests/test_utils_conf.py | 72 +--------- tox.ini | 4 +- 22 files changed, 103 insertions(+), 587 deletions(-) diff --git a/conftest.py b/conftest.py index 2ab3dffd4..77b0e033b 100644 --- a/conftest.py +++ b/conftest.py @@ -89,6 +89,30 @@ def requires_uvloop(request): pytest.skip("uvloop is not installed") +@pytest.fixture(autouse=True) +def requires_botocore(request): + if not request.node.get_closest_marker("requires_botocore"): + return + try: + import botocore + + del botocore + except ImportError: + pytest.skip("botocore is not installed") + + +@pytest.fixture(autouse=True) +def requires_boto3(request): + if not request.node.get_closest_marker("requires_boto3"): + return + try: + import boto3 + + del boto3 + except ImportError: + pytest.skip("boto3 is not installed") + + def pytest_configure(config): if config.getoption("--reactor") == "asyncio": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 6eb4af9bd..6ffb8ae93 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -278,8 +278,6 @@ Supported options: * ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) -* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O`` - Usage examples:: $ scrapy crawl myspider @@ -291,9 +289,6 @@ Usage examples:: $ scrapy crawl -O myfile:json myspider [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ] - $ scrapy crawl -o myfile -t csv myspider - [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] - .. command:: check check diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion index e2f2dc82b..82eb77cc0 100644 --- a/extras/scrapy_zsh_completion +++ b/extras/scrapy_zsh_completion @@ -41,7 +41,6 @@ _scrapy() { (runspider) local options=( {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' - {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)' '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' '1:spider file:_files -g \*.py' ) @@ -99,7 +98,6 @@ _scrapy() { (crawl) local options=( {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' - {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)' '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' '1:spider:_scrapy_spiders' ) diff --git a/pytest.ini b/pytest.ini index 16983be5e..824c0e9e9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -21,8 +21,6 @@ markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed requires_uvloop: marks tests as only enabled when uvloop is known to be working + requires_botocore: marks tests that need botocore (but not boto3) + requires_boto3: marks tests that need botocore and boto3 filterwarnings = - ignore:scrapy.downloadermiddlewares.decompression is deprecated - ignore:Module scrapy.utils.reqser is deprecated - ignore:typing.re is deprecated - ignore:typing.io is deprecated diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index eccbef040..56199cc01 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -162,12 +162,6 @@ class BaseRunSpiderCommand(ScrapyCommand): help="dump scraped items into FILE, overwriting any existing file," " to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)", ) - parser.add_argument( - "-t", - "--output-format", - metavar="FORMAT", - help="format to use for dumping items", - ) def process_options(self, args: list[str], opts: argparse.Namespace) -> None: super().process_options(args, opts) @@ -179,8 +173,7 @@ class BaseRunSpiderCommand(ScrapyCommand): feeds = feed_process_params_from_cli( self.settings, opts.output, - opts.output_format, - opts.overwrite_output, + overwrite_output=opts.overwrite_output, ) self.settings.set("FEEDS", feeds, priority="cmdline") diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 3e5657d22..de0cf543e 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging import pprint import signal -import warnings from typing import TYPE_CHECKING, Any, TypeVar, cast from twisted.internet.defer import ( @@ -17,7 +16,6 @@ from zope.interface.verify import verifyClass from scrapy import Spider, signals from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.logformatter import LogFormatter @@ -142,10 +140,8 @@ class Crawler: if self.crawling: raise RuntimeError("Crawling already taking place") if self._started: - warnings.warn( - "Running Crawler.crawl() more than once is deprecated.", - ScrapyDeprecationWarning, - stacklevel=2, + raise RuntimeError( + "Cannot run Crawler.crawl() more than once on the same instance." ) self.crawling = self._started = True diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 84678b8e9..a65757972 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,5 @@ from __future__ import annotations -import warnings from itertools import chain from logging import getLogger from typing import TYPE_CHECKING, Any @@ -15,7 +14,6 @@ from scrapy.utils._compression import ( _unbrotli, _unzstd, ) -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip if TYPE_CHECKING: @@ -72,21 +70,7 @@ class HttpCompressionMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("COMPRESSION_ENABLED"): raise NotConfigured - try: - return cls(crawler=crawler) - except TypeError: - warnings.warn( - "HttpCompressionMiddleware subclasses must either modify " - "their '__init__' method to support a 'crawler' parameter or " - "reimplement their 'from_crawler' method.", - ScrapyDeprecationWarning, - ) - mw = cls() - mw.stats = crawler.stats - mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") - mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") - crawler.signals.connect(mw.open_spider, signals.spider_opened) - return mw + return cls(crawler=crawler) def open_spider(self, spider: Spider) -> None: if hasattr(spider, "download_maxsize"): diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 7c0e2280c..9fab172a8 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -12,12 +12,10 @@ once the spider has finished crawling all regular (non-failed) pages. from __future__ import annotations -import warnings from logging import Logger, getLogger -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.settings import BaseSettings, Settings +from scrapy.exceptions import NotConfigured from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message @@ -29,33 +27,13 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.http.request import Request + from scrapy.settings import BaseSettings from scrapy.spiders import Spider retry_logger = getLogger(__name__) -def backwards_compatibility_getattr(self: Any, name: str) -> tuple[Any, ...]: - if name == "EXCEPTIONS_TO_RETRY": - warnings.warn( - "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " - "Use the RETRY_EXCEPTIONS setting instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return tuple( - load_object(x) if isinstance(x, str) else x - for x in Settings().getlist("RETRY_EXCEPTIONS") - ) - raise AttributeError( - f"{self.__class__.__name__!r} object has no attribute {name!r}" - ) - - -class BackwardsCompatibilityMetaclass(type): - __getattr__ = backwards_compatibility_getattr - - def get_retry_request( request: Request, *, @@ -144,22 +122,17 @@ def get_retry_request( return None -class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): +class RetryMiddleware: def __init__(self, settings: BaseSettings): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured self.max_retry_times = settings.getint("RETRY_TIMES") self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") - - try: - self.exceptions_to_retry = self.__getattribute__("EXCEPTIONS_TO_RETRY") - except AttributeError: - # If EXCEPTIONS_TO_RETRY is not "overridden" - self.exceptions_to_retry = tuple( - load_object(x) if isinstance(x, str) else x - for x in settings.getlist("RETRY_EXCEPTIONS") - ) + self.exceptions_to_retry = tuple( + load_object(x) if isinstance(x, str) else x + for x in settings.getlist("RETRY_EXCEPTIONS") + ) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -199,5 +172,3 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): max_retry_times=max_retry_times, priority_adjust=priority_adjust, ) - - __getattr__ = backwards_compatibility_getattr diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index eb1698ce5..6ab88dbb4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -26,10 +26,8 @@ from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.settings import Settings -from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import maybe_deferred_to_future -from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import build_from_crawler, load_object @@ -48,13 +46,6 @@ if TYPE_CHECKING: from scrapy.exporters import BaseItemExporter from scrapy.settings import BaseSettings -try: - import boto3 # noqa: F401 - - IS_BOTO3_AVAILABLE = True -except ImportError: - IS_BOTO3_AVAILABLE = False - logger = logging.getLogger(__name__) @@ -217,8 +208,10 @@ class S3FeedStorage(BlockingFeedStorage): session_token: str | None = None, region_name: str | None = None, ): - if not is_botocore_available(): - raise NotConfigured("missing botocore library") + try: + import boto3.session + except ImportError: + raise NotConfigured("missing boto3 library") u = urlparse(uri) assert u.hostname self.bucketname: str = u.hostname @@ -229,42 +222,16 @@ class S3FeedStorage(BlockingFeedStorage): self.acl: str | None = acl self.endpoint_url: str | None = endpoint_url self.region_name: str | None = region_name - # It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client, - # there seems to be no good way to infer it statically. - self.s3_client: Any - if IS_BOTO3_AVAILABLE: - import boto3.session - - boto3_session = boto3.session.Session() - - self.s3_client = boto3_session.client( - "s3", - aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key, - aws_session_token=self.session_token, - endpoint_url=self.endpoint_url, - region_name=self.region_name, - ) - else: - warnings.warn( - "`botocore` usage has been deprecated for S3 feed " - "export, please use `boto3` to avoid problems", - category=ScrapyDeprecationWarning, - ) - - import botocore.session - - botocore_session = botocore.session.get_session() - - self.s3_client = botocore_session.create_client( - "s3", - aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key, - aws_session_token=self.session_token, - endpoint_url=self.endpoint_url, - region_name=self.region_name, - ) + boto3_session = boto3.session.Session() + self.s3_client = boto3_session.client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url, + region_name=self.region_name, + ) if feed_options and feed_options.get("overwrite", True) is False: logger.warning( @@ -295,17 +262,10 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - kwargs: dict[str, Any] - if IS_BOTO3_AVAILABLE: - kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} - self.s3_client.upload_fileobj( - Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs - ) - else: - kwargs = {"ACL": self.acl} if self.acl else {} - self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs - ) + kwargs: dict[str, Any] = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} + self.s3_client.upload_fileobj( + Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs + ) file.close() @@ -464,12 +424,6 @@ class FeedSlot: self._exporting = False -_FeedSlot = create_deprecated_class( - name="_FeedSlot", - new_class=FeedSlot, -) - - class FeedExporter: _pending_deferreds: list[Deferred[None]] = [] diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index bbba7d1e1..2c4c9376e 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -8,14 +8,13 @@ from __future__ import annotations import functools import hashlib -import warnings from contextlib import suppress from io import BytesIO from typing import TYPE_CHECKING, Any, cast from itemadapter import ItemAdapter -from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import ( @@ -27,7 +26,7 @@ from scrapy.pipelines.files import ( _md5sum, ) from scrapy.settings import Settings -from scrapy.utils.python import get_func_args, to_bytes +from scrapy.utils.python import to_bytes if TYPE_CHECKING: from collections.abc import Callable, Iterable @@ -42,18 +41,6 @@ if TYPE_CHECKING: from scrapy.pipelines.media import FileInfoOrError, MediaPipeline -class NoimagesDrop(DropItem): - """Product with no images exception""" - - def __init__(self, *args: Any, **kwargs: Any): - warnings.warn( - "The NoimagesDrop class is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - super().__init__(*args, **kwargs) - - class ImageException(FileException): """General image error exception""" @@ -120,8 +107,6 @@ class ImagesPipeline(FilesPipeline): resolve("IMAGES_THUMBS"), self.THUMBS ) - self._deprecated_convert_image: bool | None = None - @classmethod def from_settings(cls, settings: Settings) -> Self: s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) @@ -203,49 +188,25 @@ class ImagesPipeline(FilesPipeline): f"{self.min_width}x{self.min_height})" ) - if self._deprecated_convert_image is None: - self._deprecated_convert_image = "response_body" not in get_func_args( - self.convert_image - ) - if self._deprecated_convert_image: - warnings.warn( - f"{self.__class__.__name__}.convert_image() method overridden in a deprecated way, " - "overridden method does not accept response_body argument.", - category=ScrapyDeprecationWarning, - ) - - if self._deprecated_convert_image: - image, buf = self.convert_image(orig_image) - else: - image, buf = self.convert_image( - orig_image, response_body=BytesIO(response.body) - ) + image, buf = self.convert_image( + orig_image, response_body=BytesIO(response.body) + ) yield path, image, buf for thumb_id, size in self.thumbs.items(): thumb_path = self.thumb_path( request, thumb_id, response=response, info=info, item=item ) - if self._deprecated_convert_image: - thumb_image, thumb_buf = self.convert_image(image, size) - else: - thumb_image, thumb_buf = self.convert_image(image, size, buf) + thumb_image, thumb_buf = self.convert_image(image, size, response_body=buf) yield thumb_path, thumb_image, thumb_buf def convert_image( self, image: Image.Image, size: tuple[int, int] | None = None, - response_body: BytesIO | None = None, + *, + response_body: BytesIO, ) -> tuple[Image.Image, BytesIO]: - if response_body is None: - warnings.warn( - f"{self.__class__.__name__}.convert_image() method called in a deprecated way, " - "method called without response_body argument.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if image.format in ("PNG", "WEBP") and image.mode == "RGBA": background = self._Image.new("RGBA", image.size, (255, 255, 255)) background.paste(image, image) @@ -268,7 +229,7 @@ class ImagesPipeline(FilesPipeline): except AttributeError: resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined] image.thumbnail(size, resampling_filter) - elif response_body is not None and image.format == "JPEG": + elif image.format == "JPEG": return image, response_body buf = BytesIO() diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 64cd31c4b..91a49c652 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -3,14 +3,13 @@ from __future__ import annotations import numbers import os import sys -import warnings from collections.abc import Iterable from configparser import ConfigParser from operator import itemgetter from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, cast -from scrapy.exceptions import ScrapyDeprecationWarning, UsageError +from scrapy.exceptions import UsageError from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values @@ -21,7 +20,7 @@ if TYPE_CHECKING: def build_component_list( compdict: MutableMapping[Any, Any], - custom: Any = None, + *, convert: Callable[[Any], Any] = update_classpath, ) -> list[Any]: """Compose a component list from a { class: order } dictionary.""" @@ -60,19 +59,6 @@ def build_component_list( "please provide a real number or None instead" ) - if custom is not None: - warnings.warn( - "The 'custom' attribute of build_component_list() is deprecated. " - "Please merge its value into 'compdict' manually or change your " - "code to use Settings.getwithbase().", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if isinstance(custom, (list, tuple)): - _check_components(custom) - return type(custom)(convert(c) for c in custom) # type: ignore[return-value] - compdict.update(custom) - _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] @@ -159,7 +145,7 @@ def feed_complete_default_values_from_settings( def feed_process_params_from_cli( settings: BaseSettings, output: list[str], - output_format: str | None = None, + *, overwrite_output: list[str] | None = None, ) -> dict[str, dict[str, Any]]: """ @@ -186,37 +172,9 @@ def feed_process_params_from_cli( raise UsageError( "Please use only one of -o/--output and -O/--overwrite-output" ) - if output_format: - raise UsageError( - "-t/--output-format is a deprecated command line option" - " and does not work in combination with -O/--overwrite-output." - " To specify a format please specify it after a colon at the end of the" - " output URI (i.e. -O :)." - " Example working in the tutorial: " - "scrapy crawl quotes -O quotes.json:json" - ) output = overwrite_output overwrite = True - if output_format: - if len(output) == 1: - check_valid_format(output_format) - message = ( - "The -t/--output-format command line option is deprecated in favor of " - "specifying the output format within the output URI using the -o/--output or the" - " -O/--overwrite-output option (i.e. -o/-O :). See the documentation" - " of the -o or -O option or the following examples for more information. " - "Examples working in the tutorial: " - "scrapy crawl quotes -o quotes.csv:csv or " - "scrapy crawl quotes -O quotes.json:json" - ) - warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) - return {output[0]: {"format": output_format}} - raise UsageError( - "The -t command-line option cannot be used if multiple output " - "URIs are specified" - ) - result: dict[str, dict[str, Any]] = {} for element in output: try: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index f8904a9aa..e7bd0b232 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -4,12 +4,11 @@ import asyncio import sys from contextlib import suppress from typing import TYPE_CHECKING, Any, Generic, TypeVar -from warnings import catch_warnings, filterwarnings, warn +from warnings import catch_warnings, filterwarnings from twisted.internet import asyncioreactor, error from twisted.internet.base import DelayedCall -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object if TYPE_CHECKING: @@ -79,22 +78,6 @@ def set_asyncio_event_loop_policy() -> None: _get_asyncio_event_loop_policy() -def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: - warn( - "Call to deprecated function " - "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n" - "\n" - "Please use get_event_loop, new_event_loop and set_event_loop" - " from asyncio instead, as the corresponding policy methods may lead" - " to unexpected behaviour.\n" - "This function is replaced by set_asyncio_event_loop_policy and" - " is meant to be used only when the reactor is being installed.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return _get_asyncio_event_loop_policy() - - def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: policy = asyncio.get_event_loop_policy() if sys.platform == "win32" and not isinstance( diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 82bdcb0f9..e80cbbb89 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -30,17 +30,9 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler -def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]: - for header in headers: - if header in request.headers: - yield header - yield from request.headers.getlist(header) - - _fingerprint_cache: WeakKeyDictionary[ Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes] -] -_fingerprint_cache = WeakKeyDictionary() +] = WeakKeyDictionary() def fingerprint( diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 92a201fd1..37348778c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -8,7 +8,6 @@ import warnings from pathlib import Path from typing import Any -import pytest from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn from pytest import mark, raises @@ -82,13 +81,10 @@ class CrawlerTestCase(BaseCrawlerTest): Crawler(DefaultSpider()) @inlineCallbacks - def test_crawler_crawl_twice_deprecated(self): + def test_crawler_crawl_twice_unsupported(self): crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) yield crawler.crawl() - with pytest.warns( - ScrapyDeprecationWarning, - match=r"Running Crawler.crawl\(\) more than once is deprecated", - ): + with raises(RuntimeError, match="more than once on the same instance"): yield crawler.crawl() def test_get_addon(self): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 19cea97ec..6a7597e9f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -8,6 +8,7 @@ from pathlib import Path from tempfile import mkdtemp, mkstemp from unittest import SkipTest, mock +import pytest from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal from twisted.internet import defer, error, reactor @@ -32,7 +33,7 @@ from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes -from scrapy.utils.test import get_crawler, skip_if_no_boto +from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import ( Echo, @@ -824,9 +825,9 @@ class HttpDownloadHandlerMock: return request +@pytest.mark.requires_botocore class S3AnonTestCase(unittest.TestCase): def setUp(self): - skip_if_no_boto() crawler = get_crawler() self.s3reqh = build_from_crawler( S3DownloadHandler, @@ -845,6 +846,7 @@ class S3AnonTestCase(unittest.TestCase): self.assertEqual(httpreq.url, "http://aws-publicdatasets.s3.amazonaws.com/") +@pytest.mark.requires_botocore class S3TestCase(unittest.TestCase): download_handler_cls: type = S3DownloadHandler @@ -856,7 +858,6 @@ class S3TestCase(unittest.TestCase): AWS_SECRET_ACCESS_KEY = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o" def setUp(self): - skip_if_no_boto() crawler = get_crawler() s3reqh = build_from_crawler( S3DownloadHandler, diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 7c36f748e..934af6590 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -3,7 +3,6 @@ from io import BytesIO from logging import WARNING from pathlib import Path from unittest import SkipTest, TestCase -from warnings import catch_warnings from testfixtures import LogCapture from w3lib.encoding import resolve_encoding @@ -12,7 +11,7 @@ from scrapy.downloadermiddlewares.httpcompression import ( ACCEPTED_ENCODINGS, HttpCompressionMiddleware, ) -from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider @@ -700,29 +699,3 @@ class HttpCompressionTest(TestCase): except ImportError: raise SkipTest("no zstd support (zstandard)") self._test_download_warnsize_request_meta("zstd") - - -class HttpCompressionSubclassTest(TestCase): - def test_init_missing_stats(self): - class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware): - def __init__(self): - super().__init__() - - crawler = get_crawler(Spider) - with catch_warnings(record=True) as caught_warnings: - HttpCompressionMiddlewareSubclass.from_crawler(crawler) - messages = tuple( - str(warning.message) - for warning in caught_warnings - if warning.category is ScrapyDeprecationWarning - ) - self.assertEqual( - messages, - ( - ( - "HttpCompressionMiddleware subclasses must either modify " - "their '__init__' method to support a 'crawler' parameter " - "or reimplement their 'from_crawler' method." - ), - ), - ) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 661175840..a010865ef 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,6 +1,5 @@ import logging import unittest -import warnings from testfixtures import LogCapture from twisted.internet import defer @@ -122,37 +121,6 @@ class RetryTest(unittest.TestCase): req = Request(f"http://www.scrapytest.org/{exc.__name__}") self._test_retry_exception(req, exc("foo"), mw) - def test_exception_to_retry_custom_middleware(self): - exc = ValueError - - with warnings.catch_warnings(record=True) as warns: - - class MyRetryMiddleware(RetryMiddleware): - EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,) - - self.assertEqual(len(warns), 1) - - mw2 = MyRetryMiddleware.from_crawler(self.crawler) - req = Request(f"http://www.scrapytest.org/{exc.__name__}") - req = mw2.process_exception(req, exc("foo"), self.spider) - assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 1) - - def test_exception_to_retry_custom_middleware_self(self): - class MyRetryMiddleware(RetryMiddleware): - def process_exception(self, request, exception, spider): - if isinstance(exception, self.EXCEPTIONS_TO_RETRY): - return self._retry(request, exception, spider) - - exc = OSError - mw2 = MyRetryMiddleware.from_crawler(self.crawler) - req = Request(f"http://www.scrapytest.org/{exc.__name__}") - with warnings.catch_warnings(record=True) as warns: - req = mw2.process_exception(req, exc("foo"), self.spider) - assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 1) - self.assertEqual(len(warns), 1) - def _test_retry_exception(self, req, exception, mw=None): if mw is None: mw = self.mw diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f59412ab4..790c347fb 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -37,7 +37,6 @@ from scrapy import signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( - IS_BOTO3_AVAILABLE, BlockingFeedStorage, FeedExporter, FeedSlot, @@ -50,7 +49,7 @@ from scrapy.extensions.feedexport import ( ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_boto +from scrapy.utils.test import get_crawler, mock_google_cloud_storage from tests.mockserver import MockFTPServer, MockServer from tests.spiders import ItemSpider @@ -240,10 +239,8 @@ class BlockingFeedStorageTest(unittest.TestCase): self.assertRaises(OSError, b.open, spider=spider) +@pytest.mark.requires_boto3 class S3FeedStorageTest(unittest.TestCase): - def setUp(self): - skip_if_no_boto() - def test_parse_credentials(self): aws_credentials = { "AWS_ACCESS_KEY_ID": "settings_key", @@ -292,38 +289,12 @@ class S3FeedStorageTest(unittest.TestCase): file = mock.MagicMock() - if IS_BOTO3_AVAILABLE: - storage.s3_client = mock.MagicMock() - yield storage.store(file) - self.assertEqual( - storage.s3_client.upload_fileobj.call_args, - mock.call(Bucket=bucket, Key=key, Fileobj=file), - ) - else: - from botocore.stub import Stubber - - with Stubber(storage.s3_client) as stub: - stub.add_response( - "put_object", - expected_params={ - "Body": file, - "Bucket": bucket, - "Key": key, - }, - service_response={}, - ) - - yield storage.store(file) - - stub.assert_no_pending_responses() - self.assertEqual( - file.method_calls, - [ - mock.call.seek(0), - # The call to read does not happen with Stubber - mock.call.close(), - ], - ) + storage.s3_client = mock.MagicMock() + yield storage.store(file) + self.assertEqual( + storage.s3_client.upload_fileobj.call_args, + mock.call(Bucket=bucket, Key=key, Fileobj=file), + ) def test_init_without_acl(self): storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") @@ -459,14 +430,11 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - if IS_BOTO3_AVAILABLE: - acl = ( - storage.s3_client.upload_fileobj.call_args[1] - .get("ExtraArgs", {}) - .get("ACL") - ) - else: - acl = storage.s3_client.put_object.call_args[1].get("ACL") + acl = ( + storage.s3_client.upload_fileobj.call_args[1] + .get("ExtraArgs", {}) + .get("ACL") + ) self.assertIsNone(acl) @defer.inlineCallbacks @@ -480,10 +448,7 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - if IS_BOTO3_AVAILABLE: - acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] - else: - acl = storage.s3_client.put_object.call_args[1]["ACL"] + acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] self.assertEqual(acl, "custom-acl") def test_overwrite_default(self): @@ -2647,9 +2612,9 @@ class BatchDeliveriesTest(FeedExportTestBase): crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12 ) + @pytest.mark.requires_boto3 @defer.inlineCallbacks def test_s3_export(self): - skip_if_no_boto() bucket = "mybucket" items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 6ce7fc059..47840caaa 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -11,6 +11,7 @@ from unittest import mock from urllib.parse import urlparse import attr +import pytest from itemadapter import ItemAdapter from twisted.internet import defer from twisted.trial import unittest @@ -30,7 +31,6 @@ from scrapy.utils.test import ( get_crawler, get_ftp_content_and_delete, get_gcs_content_and_delete, - skip_if_no_boto, ) from tests.mockserver import MockFTPServer @@ -507,11 +507,10 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(fs_store.basedir, str(path)) +@pytest.mark.requires_botocore class TestS3FilesStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): - skip_if_no_boto() - bucket = "mybucket" key = "export.csv" uri = f"s3://{bucket}/{key}" @@ -557,8 +556,6 @@ class TestS3FilesStore(unittest.TestCase): @defer.inlineCallbacks def test_stat(self): - skip_if_no_boto() - bucket = "mybucket" key = "export.csv" uri = f"s3://{bucket}/{key}" diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 2c3b191fe..7561e1fd4 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -1,24 +1,19 @@ from __future__ import annotations import dataclasses -import hashlib import io import random -import warnings from shutil import rmtree from tempfile import mkdtemp -from unittest.mock import patch import attr from itemadapter import ItemAdapter from twisted.trial import unittest -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.item import Field, Item -from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop +from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.settings import Settings -from scrapy.utils.python import to_bytes skip_pillow: str | None try: @@ -159,7 +154,7 @@ class ImagesPipelineTestCase(unittest.TestCase): with self.assertRaises(ImageException): next(self.pipeline.get_images(response=resp3, request=req, info=object())) - def test_get_images_new(self): + def test_get_images(self): self.pipeline.min_width = 0 self.pipeline.min_height = 0 self.pipeline.thumbs = {"small": (20, 20)} @@ -185,101 +180,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(thumb_img, thumb_img) self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue()) - def test_get_images_old(self): - self.pipeline.thumbs = {"small": (20, 20)} - orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) - resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue()) - req = Request(url="https://dev.mydeco.com/mydeco.gif") - - def overridden_convert_image(image, size=None): - im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) - return im, buf - - with patch.object(self.pipeline, "convert_image", overridden_convert_image): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - get_images_gen = self.pipeline.get_images( - response=resp, request=req, info=object() - ) - path, new_im, new_buf = next(get_images_gen) - self.assertEqual( - path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" - ) - self.assertEqual(orig_im.mode, new_im.mode) - self.assertEqual(orig_im.getcolors(), new_im.getcolors()) - self.assertEqual(buf.getvalue(), new_buf.getvalue()) - - thumb_path, thumb_img, thumb_buf = next(get_images_gen) - self.assertEqual( - thumb_path, - "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg", - ) - self.assertEqual(orig_im.mode, thumb_img.mode) - self.assertEqual(orig_im.getcolors(), thumb_img.getcolors()) - self.assertEqual(buf.getvalue(), thumb_buf.getvalue()) - - expected_warning_msg = ( - ".convert_image() method overridden in a deprecated way, " - "overridden method does not accept response_body argument." - ) - self.assertEqual( - len( - [ - warning - for warning in w - if expected_warning_msg in str(warning.message) - ] - ), - 1, - ) - - def test_convert_image_old(self): - # tests for old API - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - SIZE = (100, 100) - # straight forward case: RGB and JPEG - COLOUR = (0, 127, 255) - im, _ = _create_image("JPEG", "RGB", SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) - - # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) - self.assertEqual(thumbnail.mode, "RGB") - self.assertEqual(thumbnail.size, (10, 10)) - - # transparency case: RGBA and PNG - COLOUR = (0, 127, 255, 50) - im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # transparency case with palette: P and PNG - COLOUR = (0, 127, 255, 50) - im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR) - im = im.convert("P") - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # ensure that we received deprecation warnings - expected_warning_msg = ".convert_image() method called in a deprecated way" - self.assertTrue( - len( - [ - warning - for warning in w - if expected_warning_msg in str(warning.message) - ] - ) - == 4 - ) - - def test_convert_image_new(self): - # tests for new API + def test_convert_image(self): SIZE = (100, 100) # straight forward case: RGB and JPEG COLOUR = (0, 127, 255) @@ -313,19 +214,6 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) -class DeprecatedImagesPipeline(ImagesPipeline): - def file_key(self, url): - return self.image_key(url) - - def image_key(self, url): - image_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return f"empty/{image_guid}.jpg" - - def thumb_key(self, url, thumb_id): - thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return f"thumbsup/{thumb_id}/{thumb_guid}.jpg" - - class ImagesPipelineTestCaseFieldsMixin: skip = skip_pillow @@ -627,23 +515,6 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value) -class NoimagesDropTestCase(unittest.TestCase): - def test_deprecation_warning(self): - arg = "" - with warnings.catch_warnings(record=True) as w: - NoimagesDrop(arg) - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, ScrapyDeprecationWarning) - with warnings.catch_warnings(record=True) as w: - - class SubclassedNoimagesDrop(NoimagesDrop): - pass - - SubclassedNoimagesDrop(arg) - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, ScrapyDeprecationWarning) - - def _create_image(format, *a, **kw): buf = io.BytesIO() Image.new(*a, **kw).save(buf, format) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index dc3f01d57..2ce7948eb 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,9 +1,6 @@ import unittest -import warnings -import pytest - -from scrapy.exceptions import ScrapyDeprecationWarning, UsageError +from scrapy.exceptions import UsageError from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( arglist_to_dict, @@ -20,50 +17,6 @@ class BuildComponentListTest(unittest.TestCase): build_component_list(d, convert=lambda x: x), ["one", "four", "three"] ) - def test_backward_compatible_build_dict(self): - base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None} - custom = {"two": None, "three": 8, "four": 4} - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - self.assertEqual( - build_component_list(base, custom, convert=lambda x: x), - ["one", "four", "five", "three"], - ) - - def test_return_list(self): - custom = ["a", "b", "c"] - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - self.assertEqual( - build_component_list(None, custom, convert=lambda x: x), custom - ) - - def test_map_dict(self): - custom = {"one": 1, "two": 2, "three": 3} - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - self.assertEqual( - build_component_list({}, custom, convert=lambda x: x.upper()), - ["ONE", "TWO", "THREE"], - ) - - def test_map_list(self): - custom = ["a", "b", "c"] - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - self.assertEqual( - build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"] - ) - - def test_duplicate_components_in_dict(self): - duplicate_dict = {"one": 1, "two": 2, "ONE": 4} - with self.assertRaises(ValueError): - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - build_component_list({}, duplicate_dict, convert=lambda x: x.lower()) - - def test_duplicate_components_in_list(self): - duplicate_list = ["a", "b", "a"] - with self.assertRaises(ValueError) as cm: - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - build_component_list(None, duplicate_list, convert=lambda x: x) - self.assertIn(str(duplicate_list), str(cm.exception)) - def test_duplicate_components_in_basesettings(self): # Higher priority takes precedence duplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0) @@ -92,11 +45,6 @@ class BuildComponentListTest(unittest.TestCase): "c": 22222222222222222222, } self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"]) - # raise exception for invalid values - d = {"one": "5"} - with self.assertRaises(ValueError): - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - build_component_list({}, d, convert=lambda x: x) class UtilsConfTestCase(unittest.TestCase): @@ -115,7 +63,6 @@ class FeedExportConfigTestCase(unittest.TestCase): feed_process_params_from_cli, settings, ["items.dat"], - "noformat", ) def test_feed_export_config_mismatch(self): @@ -125,18 +72,8 @@ class FeedExportConfigTestCase(unittest.TestCase): feed_process_params_from_cli, settings, ["items1.dat", "items2.dat"], - "noformat", ) - def test_feed_export_config_backward_compatible(self): - with warnings.catch_warnings(record=True) as cw: - settings = Settings() - self.assertEqual( - {"items.dat": {"format": "csv"}}, - feed_process_params_from_cli(settings, ["items.dat"], "csv"), - ) - self.assertEqual(cw[0].category, ScrapyDeprecationWarning) - def test_feed_export_config_explicit_formats(self): settings = Settings() self.assertEqual( @@ -174,7 +111,9 @@ class FeedExportConfigTestCase(unittest.TestCase): settings = Settings() self.assertEqual( {"output.json": {"format": "json", "overwrite": True}}, - feed_process_params_from_cli(settings, [], None, ["output.json"]), + feed_process_params_from_cli( + settings, [], overwrite_output=["output.json"] + ), ) def test_output_and_overwrite_output(self): @@ -182,8 +121,7 @@ class FeedExportConfigTestCase(unittest.TestCase): feed_process_params_from_cli( Settings(), ["output1.json"], - None, - ["output2.json"], + overwrite_output=["output2.json"], ) def test_feed_complete_default_values_from_settings_empty(self): diff --git a/tox.ini b/tox.ini index a526fc120..5783a0e61 100644 --- a/tox.ini +++ b/tox.ini @@ -241,7 +241,7 @@ deps = {[testenv]deps} botocore>=1.4.87 commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} [testenv:botocore-pinned] basepython = {[pinned]basepython} @@ -252,4 +252,4 @@ install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore}