RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Deprecation removals. (#6500) 

* Deprecation removals.

* Clean up the default pytest filterwarnings.

* Remove test_get_images_old().

* Redo boto-requiring test filtering.

* Remove an unused function.

* Improve the Crawler.crawl() error message.

* Fix the test.
This commit is contained in:
Andrey Rakhmatullin 2024-10-31 18:06:22 +05:00 committed by GitHub
parent d2bdbad8c8
commit d85c39f5bc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 103 additions and 587 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
conftest.py
View File

@ -89,6 +89,30 @@ def requires_uvloop(request):
pytest.skip("uvloop is not installed") pytest.skip("uvloop is not installed")
@pytest.fixture(autouse=True)
def requires_botocore(request):
if not request.node.get_closest_marker("requires_botocore"):
return
try:
import botocore
del botocore
except ImportError:
pytest.skip("botocore is not installed")
@pytest.fixture(autouse=True)
def requires_boto3(request):
if not request.node.get_closest_marker("requires_boto3"):
return
try:
import boto3
del boto3
except ImportError:
pytest.skip("boto3 is not installed")
def pytest_configure(config): def pytest_configure(config):
if config.getoption("--reactor") == "asyncio": if config.getoption("--reactor") == "asyncio":
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")

5
docs/topics/commands.rst
View File

@ -278,8 +278,6 @@ Supported options:
* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) * ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``)
* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O``
Usage examples:: Usage examples::
$ scrapy crawl myspider $ scrapy crawl myspider
@ -291,9 +289,6 @@ Usage examples::
$ scrapy crawl -O myfile:json myspider $ scrapy crawl -O myfile:json myspider
[ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ] [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ]
$ scrapy crawl -o myfile -t csv myspider
[ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
.. command:: check .. command:: check
check check

2
extras/scrapy_zsh_completion
View File

@ -41,7 +41,6 @@ _scrapy() {
(runspider) (runspider)
local options=( local options=(
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider file:_files -g \*.py' '1:spider file:_files -g \*.py'
) )
@ -99,7 +98,6 @@ _scrapy() {
(crawl) (crawl)
local options=( local options=(
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider:_scrapy_spiders' '1:spider:_scrapy_spiders'
) )

6
pytest.ini
View File

@ -21,8 +21,6 @@ markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
requires_uvloop: marks tests as only enabled when uvloop is known to be working requires_uvloop: marks tests as only enabled when uvloop is known to be working
requires_botocore: marks tests that need botocore (but not boto3)
requires_boto3: marks tests that need botocore and boto3
filterwarnings = filterwarnings =
ignore:scrapy.downloadermiddlewares.decompression is deprecated
ignore:Module scrapy.utils.reqser is deprecated
ignore:typing.re is deprecated
ignore:typing.io is deprecated

9
scrapy/commands/__init__.py
View File

@ -162,12 +162,6 @@ class BaseRunSpiderCommand(ScrapyCommand):
help="dump scraped items into FILE, overwriting any existing file," help="dump scraped items into FILE, overwriting any existing file,"
" to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)", " to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)",
) )
parser.add_argument(
"-t",
"--output-format",
metavar="FORMAT",
help="format to use for dumping items",
)
def process_options(self, args: list[str], opts: argparse.Namespace) -> None: def process_options(self, args: list[str], opts: argparse.Namespace) -> None:
super().process_options(args, opts) super().process_options(args, opts)
@ -179,8 +173,7 @@ class BaseRunSpiderCommand(ScrapyCommand):
feeds = feed_process_params_from_cli( feeds = feed_process_params_from_cli(
self.settings, self.settings,
opts.output, opts.output,
opts.output_format, overwrite_output=opts.overwrite_output,
opts.overwrite_output,
) )
self.settings.set("FEEDS", feeds, priority="cmdline") self.settings.set("FEEDS", feeds, priority="cmdline")

8
scrapy/crawler.py
View File

@ -3,7 +3,6 @@ from __future__ import annotations
import logging import logging
import pprint import pprint
import signal import signal
import warnings
from typing import TYPE_CHECKING, Any, TypeVar, cast from typing import TYPE_CHECKING, Any, TypeVar, cast
from twisted.internet.defer import ( from twisted.internet.defer import (
@ -17,7 +16,6 @@ from zope.interface.verify import verifyClass
from scrapy import Spider, signals from scrapy import Spider, signals
from scrapy.addons import AddonManager from scrapy.addons import AddonManager
from scrapy.core.engine import ExecutionEngine from scrapy.core.engine import ExecutionEngine
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager from scrapy.extension import ExtensionManager
from scrapy.interfaces import ISpiderLoader from scrapy.interfaces import ISpiderLoader
from scrapy.logformatter import LogFormatter from scrapy.logformatter import LogFormatter
@ -142,10 +140,8 @@ class Crawler:
if self.crawling: if self.crawling:
raise RuntimeError("Crawling already taking place") raise RuntimeError("Crawling already taking place")
if self._started: if self._started:
warnings.warn( raise RuntimeError(
"Running Crawler.crawl() more than once is deprecated.", "Cannot run Crawler.crawl() more than once on the same instance."
ScrapyDeprecationWarning,
stacklevel=2,
) )
self.crawling = self._started = True self.crawling = self._started = True

18
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -1,6 +1,5 @@
from __future__ import annotations from __future__ import annotations
import warnings
from itertools import chain from itertools import chain
from logging import getLogger from logging import getLogger
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
@ -15,7 +14,6 @@ from scrapy.utils._compression import (
_unbrotli, _unbrotli,
_unzstd, _unzstd,
) )
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.gz import gunzip from scrapy.utils.gz import gunzip
if TYPE_CHECKING: if TYPE_CHECKING:
@ -72,21 +70,7 @@ class HttpCompressionMiddleware:
def from_crawler(cls, crawler: Crawler) -> Self: def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("COMPRESSION_ENABLED"): if not crawler.settings.getbool("COMPRESSION_ENABLED"):
raise NotConfigured raise NotConfigured
try: return cls(crawler=crawler)
return cls(crawler=crawler)
except TypeError:
warnings.warn(
"HttpCompressionMiddleware subclasses must either modify "
"their '__init__' method to support a 'crawler' parameter or "
"reimplement their 'from_crawler' method.",
ScrapyDeprecationWarning,
)
mw = cls()
mw.stats = crawler.stats
mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
crawler.signals.connect(mw.open_spider, signals.spider_opened)
return mw
def open_spider(self, spider: Spider) -> None: def open_spider(self, spider: Spider) -> None:
if hasattr(spider, "download_maxsize"): if hasattr(spider, "download_maxsize"):

45
scrapy/downloadermiddlewares/retry.py
View File

@ -12,12 +12,10 @@ once the spider has finished crawling all regular (non-failed) pages.
from __future__ import annotations from __future__ import annotations
import warnings
from logging import Logger, getLogger from logging import Logger, getLogger
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exceptions import NotConfigured
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.misc import load_object from scrapy.utils.misc import load_object
from scrapy.utils.python import global_object_name from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message from scrapy.utils.response import response_status_message
@ -29,33 +27,13 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler from scrapy.crawler import Crawler
from scrapy.http import Response from scrapy.http import Response
from scrapy.http.request import Request from scrapy.http.request import Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider from scrapy.spiders import Spider
retry_logger = getLogger(__name__) retry_logger = getLogger(__name__)
def backwards_compatibility_getattr(self: Any, name: str) -> tuple[Any, ...]:
if name == "EXCEPTIONS_TO_RETRY":
warnings.warn(
"Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
"Use the RETRY_EXCEPTIONS setting instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return tuple(
load_object(x) if isinstance(x, str) else x
for x in Settings().getlist("RETRY_EXCEPTIONS")
)
raise AttributeError(
f"{self.__class__.__name__!r} object has no attribute {name!r}"
)
class BackwardsCompatibilityMetaclass(type):
__getattr__ = backwards_compatibility_getattr
def get_retry_request( def get_retry_request(
request: Request, request: Request,
*, *,
@ -144,22 +122,17 @@ def get_retry_request(
return None return None
class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): class RetryMiddleware:
def __init__(self, settings: BaseSettings): def __init__(self, settings: BaseSettings):
if not settings.getbool("RETRY_ENABLED"): if not settings.getbool("RETRY_ENABLED"):
raise NotConfigured raise NotConfigured
self.max_retry_times = settings.getint("RETRY_TIMES") self.max_retry_times = settings.getint("RETRY_TIMES")
self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}
self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
self.exceptions_to_retry = tuple(
try: load_object(x) if isinstance(x, str) else x
self.exceptions_to_retry = self.__getattribute__("EXCEPTIONS_TO_RETRY") for x in settings.getlist("RETRY_EXCEPTIONS")
except AttributeError: )
# If EXCEPTIONS_TO_RETRY is not "overridden"
self.exceptions_to_retry = tuple(
load_object(x) if isinstance(x, str) else x
for x in settings.getlist("RETRY_EXCEPTIONS")
)
@classmethod @classmethod
def from_crawler(cls, crawler: Crawler) -> Self: def from_crawler(cls, crawler: Crawler) -> Self:
@ -199,5 +172,3 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
max_retry_times=max_retry_times, max_retry_times=max_retry_times,
priority_adjust=priority_adjust, priority_adjust=priority_adjust,
) )
__getattr__ = backwards_compatibility_getattr

80
scrapy/extensions/feedexport.py
View File

@ -26,10 +26,8 @@ from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.extensions.postprocessing import PostProcessingManager
from scrapy.settings import Settings from scrapy.settings import Settings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.ftp import ftp_store_file from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.misc import build_from_crawler, load_object
@ -48,13 +46,6 @@ if TYPE_CHECKING:
from scrapy.exporters import BaseItemExporter from scrapy.exporters import BaseItemExporter
from scrapy.settings import BaseSettings from scrapy.settings import BaseSettings
try:
import boto3 # noqa: F401
IS_BOTO3_AVAILABLE = True
except ImportError:
IS_BOTO3_AVAILABLE = False
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -217,8 +208,10 @@ class S3FeedStorage(BlockingFeedStorage):
session_token: str | None = None, session_token: str | None = None,
region_name: str | None = None, region_name: str | None = None,
): ):
if not is_botocore_available(): try:
raise NotConfigured("missing botocore library") import boto3.session
except ImportError:
raise NotConfigured("missing boto3 library")
u = urlparse(uri) u = urlparse(uri)
assert u.hostname assert u.hostname
self.bucketname: str = u.hostname self.bucketname: str = u.hostname
@ -229,42 +222,16 @@ class S3FeedStorage(BlockingFeedStorage):
self.acl: str | None = acl self.acl: str | None = acl
self.endpoint_url: str | None = endpoint_url self.endpoint_url: str | None = endpoint_url
self.region_name: str | None = region_name self.region_name: str | None = region_name
# It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client,
# there seems to be no good way to infer it statically.
self.s3_client: Any
if IS_BOTO3_AVAILABLE: boto3_session = boto3.session.Session()
import boto3.session self.s3_client = boto3_session.client(
"s3",
boto3_session = boto3.session.Session() aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
self.s3_client = boto3_session.client( aws_session_token=self.session_token,
"s3", endpoint_url=self.endpoint_url,
aws_access_key_id=self.access_key, region_name=self.region_name,
aws_secret_access_key=self.secret_key, )
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
region_name=self.region_name,
)
else:
warnings.warn(
"`botocore` usage has been deprecated for S3 feed "
"export, please use `boto3` to avoid problems",
category=ScrapyDeprecationWarning,
)
import botocore.session
botocore_session = botocore.session.get_session()
self.s3_client = botocore_session.create_client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
region_name=self.region_name,
)
if feed_options and feed_options.get("overwrite", True) is False: if feed_options and feed_options.get("overwrite", True) is False:
logger.warning( logger.warning(
@ -295,17 +262,10 @@ class S3FeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file: IO[bytes]) -> None: def _store_in_thread(self, file: IO[bytes]) -> None:
file.seek(0) file.seek(0)
kwargs: dict[str, Any] kwargs: dict[str, Any] = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
if IS_BOTO3_AVAILABLE: self.s3_client.upload_fileobj(
kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
self.s3_client.upload_fileobj( )
Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
)
else:
kwargs = {"ACL": self.acl} if self.acl else {}
self.s3_client.put_object(
Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
)
file.close() file.close()
@ -464,12 +424,6 @@ class FeedSlot:
self._exporting = False self._exporting = False
_FeedSlot = create_deprecated_class(
name="_FeedSlot",
new_class=FeedSlot,
)
class FeedExporter: class FeedExporter:
_pending_deferreds: list[Deferred[None]] = [] _pending_deferreds: list[Deferred[None]] = []

57
scrapy/pipelines/images.py
View File

@ -8,14 +8,13 @@ from __future__ import annotations
import functools import functools
import hashlib import hashlib
import warnings
from contextlib import suppress from contextlib import suppress
from io import BytesIO from io import BytesIO
from typing import TYPE_CHECKING, Any, cast from typing import TYPE_CHECKING, Any, cast
from itemadapter import ItemAdapter from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import ( from scrapy.pipelines.files import (
@ -27,7 +26,7 @@ from scrapy.pipelines.files import (
_md5sum, _md5sum,
) )
from scrapy.settings import Settings from scrapy.settings import Settings
from scrapy.utils.python import get_func_args, to_bytes from scrapy.utils.python import to_bytes
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Callable, Iterable from collections.abc import Callable, Iterable
@ -42,18 +41,6 @@ if TYPE_CHECKING:
from scrapy.pipelines.media import FileInfoOrError, MediaPipeline from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
class NoimagesDrop(DropItem):
"""Product with no images exception"""
def __init__(self, *args: Any, **kwargs: Any):
warnings.warn(
"The NoimagesDrop class is deprecated",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
super().__init__(*args, **kwargs)
class ImageException(FileException): class ImageException(FileException):
"""General image error exception""" """General image error exception"""
@ -120,8 +107,6 @@ class ImagesPipeline(FilesPipeline):
resolve("IMAGES_THUMBS"), self.THUMBS resolve("IMAGES_THUMBS"), self.THUMBS
) )
self._deprecated_convert_image: bool | None = None
@classmethod @classmethod
def from_settings(cls, settings: Settings) -> Self: def from_settings(cls, settings: Settings) -> Self:
s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"])
@ -203,49 +188,25 @@ class ImagesPipeline(FilesPipeline):
f"{self.min_width}x{self.min_height})" f"{self.min_width}x{self.min_height})"
) )
if self._deprecated_convert_image is None: image, buf = self.convert_image(
self._deprecated_convert_image = "response_body" not in get_func_args( orig_image, response_body=BytesIO(response.body)
self.convert_image )
)
if self._deprecated_convert_image:
warnings.warn(
f"{self.__class__.__name__}.convert_image() method overridden in a deprecated way, "
"overridden method does not accept response_body argument.",
category=ScrapyDeprecationWarning,
)
if self._deprecated_convert_image:
image, buf = self.convert_image(orig_image)
else:
image, buf = self.convert_image(
orig_image, response_body=BytesIO(response.body)
)
yield path, image, buf yield path, image, buf
for thumb_id, size in self.thumbs.items(): for thumb_id, size in self.thumbs.items():
thumb_path = self.thumb_path( thumb_path = self.thumb_path(
request, thumb_id, response=response, info=info, item=item request, thumb_id, response=response, info=info, item=item
) )
if self._deprecated_convert_image: thumb_image, thumb_buf = self.convert_image(image, size, response_body=buf)
thumb_image, thumb_buf = self.convert_image(image, size)
else:
thumb_image, thumb_buf = self.convert_image(image, size, buf)
yield thumb_path, thumb_image, thumb_buf yield thumb_path, thumb_image, thumb_buf
def convert_image( def convert_image(
self, self,
image: Image.Image, image: Image.Image,
size: tuple[int, int] | None = None, size: tuple[int, int] | None = None,
response_body: BytesIO | None = None, *,
response_body: BytesIO,
) -> tuple[Image.Image, BytesIO]: ) -> tuple[Image.Image, BytesIO]:
if response_body is None:
warnings.warn(
f"{self.__class__.__name__}.convert_image() method called in a deprecated way, "
"method called without response_body argument.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if image.format in ("PNG", "WEBP") and image.mode == "RGBA": if image.format in ("PNG", "WEBP") and image.mode == "RGBA":
background = self._Image.new("RGBA", image.size, (255, 255, 255)) background = self._Image.new("RGBA", image.size, (255, 255, 255))
background.paste(image, image) background.paste(image, image)
@ -268,7 +229,7 @@ class ImagesPipeline(FilesPipeline):
except AttributeError: except AttributeError:
resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined] resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined]
image.thumbnail(size, resampling_filter) image.thumbnail(size, resampling_filter)
elif response_body is not None and image.format == "JPEG": elif image.format == "JPEG":
return image, response_body return image, response_body
buf = BytesIO() buf = BytesIO()

48
scrapy/utils/conf.py
View File

@ -3,14 +3,13 @@ from __future__ import annotations
import numbers import numbers
import os import os
import sys import sys
import warnings
from collections.abc import Iterable from collections.abc import Iterable
from configparser import ConfigParser from configparser import ConfigParser
from operator import itemgetter from operator import itemgetter
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, cast from typing import TYPE_CHECKING, Any, Callable, cast
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.exceptions import UsageError
from scrapy.settings import BaseSettings from scrapy.settings import BaseSettings
from scrapy.utils.deprecate import update_classpath from scrapy.utils.deprecate import update_classpath
from scrapy.utils.python import without_none_values from scrapy.utils.python import without_none_values
@ -21,7 +20,7 @@ if TYPE_CHECKING:
def build_component_list( def build_component_list(
compdict: MutableMapping[Any, Any], compdict: MutableMapping[Any, Any],
custom: Any = None, *,
convert: Callable[[Any], Any] = update_classpath, convert: Callable[[Any], Any] = update_classpath,
) -> list[Any]: ) -> list[Any]:
"""Compose a component list from a { class: order } dictionary.""" """Compose a component list from a { class: order } dictionary."""
@ -60,19 +59,6 @@ def build_component_list(
"please provide a real number or None instead" "please provide a real number or None instead"
) )
if custom is not None:
warnings.warn(
"The 'custom' attribute of build_component_list() is deprecated. "
"Please merge its value into 'compdict' manually or change your "
"code to use Settings.getwithbase().",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if isinstance(custom, (list, tuple)):
_check_components(custom)
return type(custom)(convert(c) for c in custom) # type: ignore[return-value]
compdict.update(custom)
_validate_values(compdict) _validate_values(compdict)
compdict = without_none_values(_map_keys(compdict)) compdict = without_none_values(_map_keys(compdict))
return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
@ -159,7 +145,7 @@ def feed_complete_default_values_from_settings(
def feed_process_params_from_cli( def feed_process_params_from_cli(
settings: BaseSettings, settings: BaseSettings,
output: list[str], output: list[str],
output_format: str | None = None, *,
overwrite_output: list[str] | None = None, overwrite_output: list[str] | None = None,
) -> dict[str, dict[str, Any]]: ) -> dict[str, dict[str, Any]]:
""" """
@ -186,37 +172,9 @@ def feed_process_params_from_cli(
raise UsageError( raise UsageError(
"Please use only one of -o/--output and -O/--overwrite-output" "Please use only one of -o/--output and -O/--overwrite-output"
) )
if output_format:
raise UsageError(
"-t/--output-format is a deprecated command line option"
" and does not work in combination with -O/--overwrite-output."
" To specify a format please specify it after a colon at the end of the"
" output URI (i.e. -O <URI>:<FORMAT>)."
" Example working in the tutorial: "
"scrapy crawl quotes -O quotes.json:json"
)
output = overwrite_output output = overwrite_output
overwrite = True overwrite = True
if output_format:
if len(output) == 1:
check_valid_format(output_format)
message = (
"The -t/--output-format command line option is deprecated in favor of "
"specifying the output format within the output URI using the -o/--output or the"
" -O/--overwrite-output option (i.e. -o/-O <URI>:<FORMAT>). See the documentation"
" of the -o or -O option or the following examples for more information. "
"Examples working in the tutorial: "
"scrapy crawl quotes -o quotes.csv:csv or "
"scrapy crawl quotes -O quotes.json:json"
)
warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
return {output[0]: {"format": output_format}}
raise UsageError(
"The -t command-line option cannot be used if multiple output "
"URIs are specified"
)
result: dict[str, dict[str, Any]] = {} result: dict[str, dict[str, Any]] = {}
for element in output: for element in output:
try: try:

19
scrapy/utils/reactor.py
View File

@ -4,12 +4,11 @@ import asyncio
import sys import sys
from contextlib import suppress from contextlib import suppress
from typing import TYPE_CHECKING, Any, Generic, TypeVar from typing import TYPE_CHECKING, Any, Generic, TypeVar
from warnings import catch_warnings, filterwarnings, warn from warnings import catch_warnings, filterwarnings
from twisted.internet import asyncioreactor, error from twisted.internet import asyncioreactor, error
from twisted.internet.base import DelayedCall from twisted.internet.base import DelayedCall
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.misc import load_object from scrapy.utils.misc import load_object
if TYPE_CHECKING: if TYPE_CHECKING:
@ -79,22 +78,6 @@ def set_asyncio_event_loop_policy() -> None:
_get_asyncio_event_loop_policy() _get_asyncio_event_loop_policy()
def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:
warn(
"Call to deprecated function "
"scrapy.utils.reactor.get_asyncio_event_loop_policy().\n"
"\n"
"Please use get_event_loop, new_event_loop and set_event_loop"
" from asyncio instead, as the corresponding policy methods may lead"
" to unexpected behaviour.\n"
"This function is replaced by set_asyncio_event_loop_policy and"
" is meant to be used only when the reactor is being installed.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return _get_asyncio_event_loop_policy()
def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy:
policy = asyncio.get_event_loop_policy() policy = asyncio.get_event_loop_policy()
if sys.platform == "win32" and not isinstance( if sys.platform == "win32" and not isinstance(

10
scrapy/utils/request.py
View File

@ -30,17 +30,9 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler from scrapy.crawler import Crawler
def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]:
for header in headers:
if header in request.headers:
yield header
yield from request.headers.getlist(header)
_fingerprint_cache: WeakKeyDictionary[ _fingerprint_cache: WeakKeyDictionary[
Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes] Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]
] ] = WeakKeyDictionary()
_fingerprint_cache = WeakKeyDictionary()
def fingerprint( def fingerprint(

8
tests/test_crawler.py
View File

@ -8,7 +8,6 @@ import warnings
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
import pytest
from packaging.version import parse as parse_version from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn from pexpect.popen_spawn import PopenSpawn
from pytest import mark, raises from pytest import mark, raises
@ -82,13 +81,10 @@ class CrawlerTestCase(BaseCrawlerTest):
Crawler(DefaultSpider()) Crawler(DefaultSpider())
@inlineCallbacks @inlineCallbacks
def test_crawler_crawl_twice_deprecated(self): def test_crawler_crawl_twice_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
yield crawler.crawl() yield crawler.crawl()
with pytest.warns( with raises(RuntimeError, match="more than once on the same instance"):
ScrapyDeprecationWarning,
match=r"Running Crawler.crawl\(\) more than once is deprecated",
):
yield crawler.crawl() yield crawler.crawl()
def test_get_addon(self): def test_get_addon(self):

7
tests/test_downloader_handlers.py
View File

@ -8,6 +8,7 @@ from pathlib import Path
from tempfile import mkdtemp, mkstemp from tempfile import mkdtemp, mkstemp
from unittest import SkipTest, mock from unittest import SkipTest, mock
import pytest
from testfixtures import LogCapture from testfixtures import LogCapture
from twisted.cred import checkers, credentials, portal from twisted.cred import checkers, credentials, portal
from twisted.internet import defer, error, reactor from twisted.internet import defer, error, reactor
@ -32,7 +33,7 @@ from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver import ( from tests.mockserver import (
Echo, Echo,
@ -824,9 +825,9 @@ class HttpDownloadHandlerMock:
return request return request
@pytest.mark.requires_botocore
class S3AnonTestCase(unittest.TestCase): class S3AnonTestCase(unittest.TestCase):
def setUp(self): def setUp(self):
skip_if_no_boto()
crawler = get_crawler() crawler = get_crawler()
self.s3reqh = build_from_crawler( self.s3reqh = build_from_crawler(
S3DownloadHandler, S3DownloadHandler,
@ -845,6 +846,7 @@ class S3AnonTestCase(unittest.TestCase):
self.assertEqual(httpreq.url, "http://aws-publicdatasets.s3.amazonaws.com/") self.assertEqual(httpreq.url, "http://aws-publicdatasets.s3.amazonaws.com/")
@pytest.mark.requires_botocore
class S3TestCase(unittest.TestCase): class S3TestCase(unittest.TestCase):
download_handler_cls: type = S3DownloadHandler download_handler_cls: type = S3DownloadHandler
@ -856,7 +858,6 @@ class S3TestCase(unittest.TestCase):
AWS_SECRET_ACCESS_KEY = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o" AWS_SECRET_ACCESS_KEY = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o"
def setUp(self): def setUp(self):
skip_if_no_boto()
crawler = get_crawler() crawler = get_crawler()
s3reqh = build_from_crawler( s3reqh = build_from_crawler(
S3DownloadHandler, S3DownloadHandler,

29
tests/test_downloadermiddleware_httpcompression.py
View File

@ -3,7 +3,6 @@ from io import BytesIO
from logging import WARNING from logging import WARNING
from pathlib import Path from pathlib import Path
from unittest import SkipTest, TestCase from unittest import SkipTest, TestCase
from warnings import catch_warnings
from testfixtures import LogCapture from testfixtures import LogCapture
from w3lib.encoding import resolve_encoding from w3lib.encoding import resolve_encoding
@ -12,7 +11,7 @@ from scrapy.downloadermiddlewares.httpcompression import (
ACCEPTED_ENCODINGS, ACCEPTED_ENCODINGS,
HttpCompressionMiddleware, HttpCompressionMiddleware,
) )
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import HtmlResponse, Request, Response from scrapy.http import HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider from scrapy.spiders import Spider
@ -700,29 +699,3 @@ class HttpCompressionTest(TestCase):
except ImportError: except ImportError:
raise SkipTest("no zstd support (zstandard)") raise SkipTest("no zstd support (zstandard)")
self._test_download_warnsize_request_meta("zstd") self._test_download_warnsize_request_meta("zstd")
class HttpCompressionSubclassTest(TestCase):
def test_init_missing_stats(self):
class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):
def __init__(self):
super().__init__()
crawler = get_crawler(Spider)
with catch_warnings(record=True) as caught_warnings:
HttpCompressionMiddlewareSubclass.from_crawler(crawler)
messages = tuple(
str(warning.message)
for warning in caught_warnings
if warning.category is ScrapyDeprecationWarning
)
self.assertEqual(
messages,
(
(
"HttpCompressionMiddleware subclasses must either modify "
"their '__init__' method to support a 'crawler' parameter "
"or reimplement their 'from_crawler' method."
),
),
)

32
tests/test_downloadermiddleware_retry.py
View File

@ -1,6 +1,5 @@
import logging import logging
import unittest import unittest
import warnings
from testfixtures import LogCapture from testfixtures import LogCapture
from twisted.internet import defer from twisted.internet import defer
@ -122,37 +121,6 @@ class RetryTest(unittest.TestCase):
req = Request(f"http://www.scrapytest.org/{exc.__name__}") req = Request(f"http://www.scrapytest.org/{exc.__name__}")
self._test_retry_exception(req, exc("foo"), mw) self._test_retry_exception(req, exc("foo"), mw)
def test_exception_to_retry_custom_middleware(self):
exc = ValueError
with warnings.catch_warnings(record=True) as warns:
class MyRetryMiddleware(RetryMiddleware):
EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,)
self.assertEqual(len(warns), 1)
mw2 = MyRetryMiddleware.from_crawler(self.crawler)
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
req = mw2.process_exception(req, exc("foo"), self.spider)
assert isinstance(req, Request)
self.assertEqual(req.meta["retry_times"], 1)
def test_exception_to_retry_custom_middleware_self(self):
class MyRetryMiddleware(RetryMiddleware):
def process_exception(self, request, exception, spider):
if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
return self._retry(request, exception, spider)
exc = OSError
mw2 = MyRetryMiddleware.from_crawler(self.crawler)
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
with warnings.catch_warnings(record=True) as warns:
req = mw2.process_exception(req, exc("foo"), self.spider)
assert isinstance(req, Request)
self.assertEqual(req.meta["retry_times"], 1)
self.assertEqual(len(warns), 1)
def _test_retry_exception(self, req, exception, mw=None): def _test_retry_exception(self, req, exception, mw=None):
if mw is None: if mw is None:
mw = self.mw mw = self.mw

65
tests/test_feedexport.py
View File

@ -37,7 +37,6 @@ from scrapy import signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.exporters import CsvItemExporter, JsonItemExporter
from scrapy.extensions.feedexport import ( from scrapy.extensions.feedexport import (
IS_BOTO3_AVAILABLE,
BlockingFeedStorage, BlockingFeedStorage,
FeedExporter, FeedExporter,
FeedSlot, FeedSlot,
@ -50,7 +49,7 @@ from scrapy.extensions.feedexport import (
) )
from scrapy.settings import Settings from scrapy.settings import Settings
from scrapy.utils.python import to_unicode from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_boto from scrapy.utils.test import get_crawler, mock_google_cloud_storage
from tests.mockserver import MockFTPServer, MockServer from tests.mockserver import MockFTPServer, MockServer
from tests.spiders import ItemSpider from tests.spiders import ItemSpider
@ -240,10 +239,8 @@ class BlockingFeedStorageTest(unittest.TestCase):
self.assertRaises(OSError, b.open, spider=spider) self.assertRaises(OSError, b.open, spider=spider)
@pytest.mark.requires_boto3
class S3FeedStorageTest(unittest.TestCase): class S3FeedStorageTest(unittest.TestCase):
def setUp(self):
skip_if_no_boto()
def test_parse_credentials(self): def test_parse_credentials(self):
aws_credentials = { aws_credentials = {
"AWS_ACCESS_KEY_ID": "settings_key", "AWS_ACCESS_KEY_ID": "settings_key",
@ -292,38 +289,12 @@ class S3FeedStorageTest(unittest.TestCase):
file = mock.MagicMock() file = mock.MagicMock()
if IS_BOTO3_AVAILABLE: storage.s3_client = mock.MagicMock()
storage.s3_client = mock.MagicMock() yield storage.store(file)
yield storage.store(file) self.assertEqual(
self.assertEqual( storage.s3_client.upload_fileobj.call_args,
storage.s3_client.upload_fileobj.call_args, mock.call(Bucket=bucket, Key=key, Fileobj=file),
mock.call(Bucket=bucket, Key=key, Fileobj=file), )
)
else:
from botocore.stub import Stubber
with Stubber(storage.s3_client) as stub:
stub.add_response(
"put_object",
expected_params={
"Body": file,
"Bucket": bucket,
"Key": key,
},
service_response={},
)
yield storage.store(file)
stub.assert_no_pending_responses()
self.assertEqual(
file.method_calls,
[
mock.call.seek(0),
# The call to read does not happen with Stubber
mock.call.close(),
],
)
def test_init_without_acl(self): def test_init_without_acl(self):
storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")
@ -459,14 +430,11 @@ class S3FeedStorageTest(unittest.TestCase):
storage.s3_client = mock.MagicMock() storage.s3_client = mock.MagicMock()
yield storage.store(BytesIO(b"test file")) yield storage.store(BytesIO(b"test file"))
if IS_BOTO3_AVAILABLE: acl = (
acl = ( storage.s3_client.upload_fileobj.call_args[1]
storage.s3_client.upload_fileobj.call_args[1] .get("ExtraArgs", {})
.get("ExtraArgs", {}) .get("ACL")
.get("ACL") )
)
else:
acl = storage.s3_client.put_object.call_args[1].get("ACL")
self.assertIsNone(acl) self.assertIsNone(acl)
@defer.inlineCallbacks @defer.inlineCallbacks
@ -480,10 +448,7 @@ class S3FeedStorageTest(unittest.TestCase):
storage.s3_client = mock.MagicMock() storage.s3_client = mock.MagicMock()
yield storage.store(BytesIO(b"test file")) yield storage.store(BytesIO(b"test file"))
if IS_BOTO3_AVAILABLE: acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
else:
acl = storage.s3_client.put_object.call_args[1]["ACL"]
self.assertEqual(acl, "custom-acl") self.assertEqual(acl, "custom-acl")
def test_overwrite_default(self): def test_overwrite_default(self):
@ -2647,9 +2612,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12 crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12
) )
@pytest.mark.requires_boto3
@defer.inlineCallbacks @defer.inlineCallbacks
def test_s3_export(self): def test_s3_export(self):
skip_if_no_boto()
bucket = "mybucket" bucket = "mybucket"
items = [ items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem({"foo": "bar1", "egg": "spam1"}),

7
tests/test_pipeline_files.py
View File

@ -11,6 +11,7 @@ from unittest import mock
from urllib.parse import urlparse from urllib.parse import urlparse
import attr import attr
import pytest
from itemadapter import ItemAdapter from itemadapter import ItemAdapter
from twisted.internet import defer from twisted.internet import defer
from twisted.trial import unittest from twisted.trial import unittest
@ -30,7 +31,6 @@ from scrapy.utils.test import (
get_crawler, get_crawler,
get_ftp_content_and_delete, get_ftp_content_and_delete,
get_gcs_content_and_delete, get_gcs_content_and_delete,
skip_if_no_boto,
) )
from tests.mockserver import MockFTPServer from tests.mockserver import MockFTPServer
@ -507,11 +507,10 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
self.assertEqual(fs_store.basedir, str(path)) self.assertEqual(fs_store.basedir, str(path))
@pytest.mark.requires_botocore
class TestS3FilesStore(unittest.TestCase): class TestS3FilesStore(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def test_persist(self): def test_persist(self):
skip_if_no_boto()
bucket = "mybucket" bucket = "mybucket"
key = "export.csv" key = "export.csv"
uri = f"s3://{bucket}/{key}" uri = f"s3://{bucket}/{key}"
@ -557,8 +556,6 @@ class TestS3FilesStore(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def test_stat(self): def test_stat(self):
skip_if_no_boto()
bucket = "mybucket" bucket = "mybucket"
key = "export.csv" key = "export.csv"
uri = f"s3://{bucket}/{key}" uri = f"s3://{bucket}/{key}"

135
tests/test_pipeline_images.py
View File

@ -1,24 +1,19 @@
from __future__ import annotations from __future__ import annotations
import dataclasses import dataclasses
import hashlib
import io import io
import random import random
import warnings
from shutil import rmtree from shutil import rmtree
from tempfile import mkdtemp from tempfile import mkdtemp
from unittest.mock import patch
import attr import attr
from itemadapter import ItemAdapter from itemadapter import ItemAdapter
from twisted.trial import unittest from twisted.trial import unittest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response from scrapy.http import Request, Response
from scrapy.item import Field, Item from scrapy.item import Field, Item
from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop from scrapy.pipelines.images import ImageException, ImagesPipeline
from scrapy.settings import Settings from scrapy.settings import Settings
from scrapy.utils.python import to_bytes
skip_pillow: str | None skip_pillow: str | None
try: try:
@ -159,7 +154,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
with self.assertRaises(ImageException): with self.assertRaises(ImageException):
next(self.pipeline.get_images(response=resp3, request=req, info=object())) next(self.pipeline.get_images(response=resp3, request=req, info=object()))
def test_get_images_new(self): def test_get_images(self):
self.pipeline.min_width = 0 self.pipeline.min_width = 0
self.pipeline.min_height = 0 self.pipeline.min_height = 0
self.pipeline.thumbs = {"small": (20, 20)} self.pipeline.thumbs = {"small": (20, 20)}
@ -185,101 +180,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
self.assertEqual(thumb_img, thumb_img) self.assertEqual(thumb_img, thumb_img)
self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue()) self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue())
def test_get_images_old(self): def test_convert_image(self):
self.pipeline.thumbs = {"small": (20, 20)}
orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue())
req = Request(url="https://dev.mydeco.com/mydeco.gif")
def overridden_convert_image(image, size=None):
im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
return im, buf
with patch.object(self.pipeline, "convert_image", overridden_convert_image):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
get_images_gen = self.pipeline.get_images(
response=resp, request=req, info=object()
)
path, new_im, new_buf = next(get_images_gen)
self.assertEqual(
path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
)
self.assertEqual(orig_im.mode, new_im.mode)
self.assertEqual(orig_im.getcolors(), new_im.getcolors())
self.assertEqual(buf.getvalue(), new_buf.getvalue())
thumb_path, thumb_img, thumb_buf = next(get_images_gen)
self.assertEqual(
thumb_path,
"thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg",
)
self.assertEqual(orig_im.mode, thumb_img.mode)
self.assertEqual(orig_im.getcolors(), thumb_img.getcolors())
self.assertEqual(buf.getvalue(), thumb_buf.getvalue())
expected_warning_msg = (
".convert_image() method overridden in a deprecated way, "
"overridden method does not accept response_body argument."
)
self.assertEqual(
len(
[
warning
for warning in w
if expected_warning_msg in str(warning.message)
]
),
1,
)
def test_convert_image_old(self):
# tests for old API
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
SIZE = (100, 100)
# straight forward case: RGB and JPEG
COLOUR = (0, 127, 255)
im, _ = _create_image("JPEG", "RGB", SIZE, COLOUR)
converted, _ = self.pipeline.convert_image(im)
self.assertEqual(converted.mode, "RGB")
self.assertEqual(converted.getcolors(), [(10000, COLOUR)])
# check that thumbnail keep image ratio
thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25))
self.assertEqual(thumbnail.mode, "RGB")
self.assertEqual(thumbnail.size, (10, 10))
# transparency case: RGBA and PNG
COLOUR = (0, 127, 255, 50)
im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR)
converted, _ = self.pipeline.convert_image(im)
self.assertEqual(converted.mode, "RGB")
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
# transparency case with palette: P and PNG
COLOUR = (0, 127, 255, 50)
im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR)
im = im.convert("P")
converted, _ = self.pipeline.convert_image(im)
self.assertEqual(converted.mode, "RGB")
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
# ensure that we received deprecation warnings
expected_warning_msg = ".convert_image() method called in a deprecated way"
self.assertTrue(
len(
[
warning
for warning in w
if expected_warning_msg in str(warning.message)
]
)
== 4
)
def test_convert_image_new(self):
# tests for new API
SIZE = (100, 100) SIZE = (100, 100)
# straight forward case: RGB and JPEG # straight forward case: RGB and JPEG
COLOUR = (0, 127, 255) COLOUR = (0, 127, 255)
@ -313,19 +214,6 @@ class ImagesPipelineTestCase(unittest.TestCase):
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
class DeprecatedImagesPipeline(ImagesPipeline):
def file_key(self, url):
return self.image_key(url)
def image_key(self, url):
image_guid = hashlib.sha1(to_bytes(url)).hexdigest()
return f"empty/{image_guid}.jpg"
def thumb_key(self, url, thumb_id):
thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest()
return f"thumbsup/{thumb_id}/{thumb_guid}.jpg"
class ImagesPipelineTestCaseFieldsMixin: class ImagesPipelineTestCaseFieldsMixin:
skip = skip_pillow skip = skip_pillow
@ -627,23 +515,6 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value) self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value)
class NoimagesDropTestCase(unittest.TestCase):
def test_deprecation_warning(self):
arg = ""
with warnings.catch_warnings(record=True) as w:
NoimagesDrop(arg)
self.assertEqual(len(w), 1)
self.assertEqual(w[0].category, ScrapyDeprecationWarning)
with warnings.catch_warnings(record=True) as w:
class SubclassedNoimagesDrop(NoimagesDrop):
pass
SubclassedNoimagesDrop(arg)
self.assertEqual(len(w), 1)
self.assertEqual(w[0].category, ScrapyDeprecationWarning)
def _create_image(format, *a, **kw): def _create_image(format, *a, **kw):
buf = io.BytesIO() buf = io.BytesIO()
Image.new(*a, **kw).save(buf, format) Image.new(*a, **kw).save(buf, format)

72
tests/test_utils_conf.py
View File

@ -1,9 +1,6 @@
import unittest import unittest
import warnings
import pytest from scrapy.exceptions import UsageError
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
from scrapy.settings import BaseSettings, Settings from scrapy.settings import BaseSettings, Settings
from scrapy.utils.conf import ( from scrapy.utils.conf import (
arglist_to_dict, arglist_to_dict,
@ -20,50 +17,6 @@ class BuildComponentListTest(unittest.TestCase):
build_component_list(d, convert=lambda x: x), ["one", "four", "three"] build_component_list(d, convert=lambda x: x), ["one", "four", "three"]
) )
def test_backward_compatible_build_dict(self):
base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None}
custom = {"two": None, "three": 8, "four": 4}
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
self.assertEqual(
build_component_list(base, custom, convert=lambda x: x),
["one", "four", "five", "three"],
)
def test_return_list(self):
custom = ["a", "b", "c"]
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
self.assertEqual(
build_component_list(None, custom, convert=lambda x: x), custom
)
def test_map_dict(self):
custom = {"one": 1, "two": 2, "three": 3}
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
self.assertEqual(
build_component_list({}, custom, convert=lambda x: x.upper()),
["ONE", "TWO", "THREE"],
)
def test_map_list(self):
custom = ["a", "b", "c"]
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
self.assertEqual(
build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"]
)
def test_duplicate_components_in_dict(self):
duplicate_dict = {"one": 1, "two": 2, "ONE": 4}
with self.assertRaises(ValueError):
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
build_component_list({}, duplicate_dict, convert=lambda x: x.lower())
def test_duplicate_components_in_list(self):
duplicate_list = ["a", "b", "a"]
with self.assertRaises(ValueError) as cm:
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
build_component_list(None, duplicate_list, convert=lambda x: x)
self.assertIn(str(duplicate_list), str(cm.exception))
def test_duplicate_components_in_basesettings(self): def test_duplicate_components_in_basesettings(self):
# Higher priority takes precedence # Higher priority takes precedence
duplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0) duplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0)
@ -92,11 +45,6 @@ class BuildComponentListTest(unittest.TestCase):
"c": 22222222222222222222, "c": 22222222222222222222,
} }
self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"]) self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"])
# raise exception for invalid values
d = {"one": "5"}
with self.assertRaises(ValueError):
with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
build_component_list({}, d, convert=lambda x: x)
class UtilsConfTestCase(unittest.TestCase): class UtilsConfTestCase(unittest.TestCase):
@ -115,7 +63,6 @@ class FeedExportConfigTestCase(unittest.TestCase):
feed_process_params_from_cli, feed_process_params_from_cli,
settings, settings,
["items.dat"], ["items.dat"],
"noformat",
) )
def test_feed_export_config_mismatch(self): def test_feed_export_config_mismatch(self):
@ -125,18 +72,8 @@ class FeedExportConfigTestCase(unittest.TestCase):
feed_process_params_from_cli, feed_process_params_from_cli,
settings, settings,
["items1.dat", "items2.dat"], ["items1.dat", "items2.dat"],
"noformat",
) )
def test_feed_export_config_backward_compatible(self):
with warnings.catch_warnings(record=True) as cw:
settings = Settings()
self.assertEqual(
{"items.dat": {"format": "csv"}},
feed_process_params_from_cli(settings, ["items.dat"], "csv"),
)
self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
def test_feed_export_config_explicit_formats(self): def test_feed_export_config_explicit_formats(self):
settings = Settings() settings = Settings()
self.assertEqual( self.assertEqual(
@ -174,7 +111,9 @@ class FeedExportConfigTestCase(unittest.TestCase):
settings = Settings() settings = Settings()
self.assertEqual( self.assertEqual(
{"output.json": {"format": "json", "overwrite": True}}, {"output.json": {"format": "json", "overwrite": True}},
feed_process_params_from_cli(settings, [], None, ["output.json"]), feed_process_params_from_cli(
settings, [], overwrite_output=["output.json"]
),
) )
def test_output_and_overwrite_output(self): def test_output_and_overwrite_output(self):
@ -182,8 +121,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
feed_process_params_from_cli( feed_process_params_from_cli(
Settings(), Settings(),
["output1.json"], ["output1.json"],
None, overwrite_output=["output2.json"],
["output2.json"],
) )
def test_feed_complete_default_values_from_settings_empty(self): def test_feed_complete_default_values_from_settings_empty(self):

4
tox.ini
View File

@ -241,7 +241,7 @@ deps =
{[testenv]deps} {[testenv]deps}
botocore>=1.4.87 botocore>=1.4.87
commands = commands =
pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore}
[testenv:botocore-pinned] [testenv:botocore-pinned]
basepython = {[pinned]basepython} basepython = {[pinned]basepython}
@ -252,4 +252,4 @@ install_command = {[pinned]install_command}
setenv = setenv =
{[pinned]setenv} {[pinned]setenv}
commands = commands =
pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore}