mirror of https://github.com/scrapy/scrapy.git
Remove code deprecated in 2.12.0 (#7126)
* Remove smaller things. * Remove from_settings(). * Remove old MediaPipeline init code. * Cleanup. * Typo.
This commit is contained in:
parent
d414d393d4
commit
61b4befc60
|
|
@ -126,7 +126,7 @@ coverage_ignore_pyobjects = [
|
||||||
# The interface methods of duplicate request filtering classes are already
|
# The interface methods of duplicate request filtering classes are already
|
||||||
# covered in the interface documentation part of the DUPEFILTER_CLASS
|
# covered in the interface documentation part of the DUPEFILTER_CLASS
|
||||||
# setting documentation.
|
# setting documentation.
|
||||||
r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_settings|request_seen|open|close|log)$",
|
r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_crawler|request_seen|open|close|log)$",
|
||||||
# Private exception used by the command-line interface implementation.
|
# Private exception used by the command-line interface implementation.
|
||||||
r"^scrapy\.exceptions\.UsageError",
|
r"^scrapy\.exceptions\.UsageError",
|
||||||
# Methods of BaseItemExporter subclasses are only documented in
|
# Methods of BaseItemExporter subclasses are only documented in
|
||||||
|
|
|
||||||
|
|
@ -1131,9 +1131,9 @@ interface::
|
||||||
class MyDupeFilter:
|
class MyDupeFilter:
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_settings(cls, settings):
|
def from_crawler(cls, crawler):
|
||||||
"""Returns an instance of this duplicate request filtering class
|
"""Returns an instance of this duplicate request filtering class
|
||||||
based on the current crawl settings."""
|
based on the current Crawler instance."""
|
||||||
return cls()
|
return cls()
|
||||||
|
|
||||||
def request_seen(self, request):
|
def request_seen(self, request):
|
||||||
|
|
|
||||||
|
|
@ -29,23 +29,6 @@ __version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").
|
||||||
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."))
|
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."))
|
||||||
|
|
||||||
|
|
||||||
def __getattr__(name: str):
|
|
||||||
if name == "twisted_version":
|
|
||||||
import warnings # noqa: PLC0415 # pylint: disable=reimported
|
|
||||||
|
|
||||||
from twisted import version as _txv # noqa: PLC0415
|
|
||||||
|
|
||||||
from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415
|
|
||||||
|
|
||||||
warnings.warn(
|
|
||||||
"The scrapy.twisted_version attribute is deprecated, use twisted.version instead",
|
|
||||||
ScrapyDeprecationWarning,
|
|
||||||
)
|
|
||||||
return _txv.major, _txv.minor, _txv.micro
|
|
||||||
|
|
||||||
raise AttributeError
|
|
||||||
|
|
||||||
|
|
||||||
# Ignore noisy twisted deprecation warnings
|
# Ignore noisy twisted deprecation warnings
|
||||||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted")
|
warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -182,14 +182,6 @@ class Downloader:
|
||||||
|
|
||||||
return key
|
return key
|
||||||
|
|
||||||
def _get_slot_key(self, request: Request, spider: Spider | None) -> str:
|
|
||||||
warnings.warn(
|
|
||||||
"Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.",
|
|
||||||
ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
return self.get_slot_key(request)
|
|
||||||
|
|
||||||
# passed as download_func into self.middleware.download() in self.fetch()
|
# passed as download_func into self.middleware.download() in self.fetch()
|
||||||
@inlineCallbacks
|
@inlineCallbacks
|
||||||
def _enqueue_request(
|
def _enqueue_request(
|
||||||
|
|
|
||||||
|
|
@ -71,21 +71,6 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
||||||
stacklevel=2,
|
stacklevel=2,
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_settings(
|
|
||||||
cls,
|
|
||||||
settings: BaseSettings,
|
|
||||||
method: int = SSL.SSLv23_METHOD,
|
|
||||||
*args: Any,
|
|
||||||
**kwargs: Any,
|
|
||||||
) -> Self:
|
|
||||||
warnings.warn(
|
|
||||||
f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
return cls._from_settings(settings, method, *args, **kwargs)
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_crawler(
|
def from_crawler(
|
||||||
cls,
|
cls,
|
||||||
|
|
@ -94,20 +79,10 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
||||||
*args: Any,
|
*args: Any,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> Self:
|
) -> Self:
|
||||||
return cls._from_settings(crawler.settings, method, *args, **kwargs)
|
tls_verbose_logging: bool = crawler.settings.getbool(
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _from_settings(
|
|
||||||
cls,
|
|
||||||
settings: BaseSettings,
|
|
||||||
method: int = SSL.SSLv23_METHOD,
|
|
||||||
*args: Any,
|
|
||||||
**kwargs: Any,
|
|
||||||
) -> Self:
|
|
||||||
tls_verbose_logging: bool = settings.getbool(
|
|
||||||
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
|
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
|
||||||
)
|
)
|
||||||
tls_ciphers: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
|
tls_ciphers: str | None = crawler.settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
|
||||||
return cls( # type: ignore[misc]
|
return cls( # type: ignore[misc]
|
||||||
method=method,
|
method=method,
|
||||||
tls_verbose_logging=tls_verbose_logging,
|
tls_verbose_logging=tls_verbose_logging,
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,6 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import warnings
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
from warnings import warn
|
from warnings import warn
|
||||||
|
|
@ -22,7 +21,6 @@ if TYPE_CHECKING:
|
||||||
|
|
||||||
from scrapy.crawler import Crawler
|
from scrapy.crawler import Crawler
|
||||||
from scrapy.http.request import Request
|
from scrapy.http.request import Request
|
||||||
from scrapy.settings import BaseSettings
|
|
||||||
from scrapy.spiders import Spider
|
from scrapy.spiders import Spider
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -30,15 +28,6 @@ class BaseDupeFilter:
|
||||||
"""Dummy duplicate request filtering class (:setting:`DUPEFILTER_CLASS`)
|
"""Dummy duplicate request filtering class (:setting:`DUPEFILTER_CLASS`)
|
||||||
that does not filter out any request."""
|
that does not filter out any request."""
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_settings(cls, settings: BaseSettings) -> Self:
|
|
||||||
warnings.warn(
|
|
||||||
f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
return cls()
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||||
return cls()
|
return cls()
|
||||||
|
|
@ -88,38 +77,16 @@ class RFPDupeFilter(BaseDupeFilter):
|
||||||
self.file.seek(0)
|
self.file.seek(0)
|
||||||
self.fingerprints.update(x.rstrip() for x in self.file)
|
self.fingerprints.update(x.rstrip() for x in self.file)
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_settings(
|
|
||||||
cls,
|
|
||||||
settings: BaseSettings,
|
|
||||||
*,
|
|
||||||
fingerprinter: RequestFingerprinterProtocol | None = None,
|
|
||||||
) -> Self:
|
|
||||||
warnings.warn(
|
|
||||||
f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
return cls._from_settings(settings, fingerprinter=fingerprinter)
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||||
assert crawler.request_fingerprinter
|
assert crawler.request_fingerprinter
|
||||||
return cls._from_settings(
|
debug = crawler.settings.getbool("DUPEFILTER_DEBUG")
|
||||||
crawler.settings,
|
return cls(
|
||||||
|
job_dir(crawler.settings),
|
||||||
|
debug,
|
||||||
fingerprinter=crawler.request_fingerprinter,
|
fingerprinter=crawler.request_fingerprinter,
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _from_settings(
|
|
||||||
cls,
|
|
||||||
settings: BaseSettings,
|
|
||||||
*,
|
|
||||||
fingerprinter: RequestFingerprinterProtocol | None = None,
|
|
||||||
) -> Self:
|
|
||||||
debug = settings.getbool("DUPEFILTER_DEBUG")
|
|
||||||
return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
|
|
||||||
|
|
||||||
def request_seen(self, request: Request) -> bool:
|
def request_seen(self, request: Request) -> bool:
|
||||||
fp = self.request_fingerprint(request)
|
fp = self.request_fingerprint(request)
|
||||||
if fp in self.fingerprints:
|
if fp in self.fingerprints:
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@ from collections.abc import Callable
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path, PureWindowsPath
|
from pathlib import Path, PureWindowsPath
|
||||||
from tempfile import NamedTemporaryFile
|
from tempfile import NamedTemporaryFile
|
||||||
from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, TypeVar, cast
|
from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, cast
|
||||||
from urllib.parse import unquote, urlparse
|
from urllib.parse import unquote, urlparse
|
||||||
|
|
||||||
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
|
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
|
||||||
|
|
@ -35,8 +35,6 @@ from scrapy.utils.misc import build_from_crawler, load_object
|
||||||
from scrapy.utils.python import without_none_values
|
from scrapy.utils.python import without_none_values
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import Iterable
|
|
||||||
|
|
||||||
from _typeshed import OpenBinaryMode
|
from _typeshed import OpenBinaryMode
|
||||||
from twisted.python.failure import Failure
|
from twisted.python.failure import Failure
|
||||||
|
|
||||||
|
|
@ -54,25 +52,6 @@ UriParamsCallableT: TypeAlias = Callable[
|
||||||
[dict[str, Any], Spider], dict[str, Any] | None
|
[dict[str, Any], Spider], dict[str, Any] | None
|
||||||
]
|
]
|
||||||
|
|
||||||
_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol")
|
|
||||||
|
|
||||||
|
|
||||||
def build_storage(
|
|
||||||
builder: Callable[..., _StorageT],
|
|
||||||
uri: str,
|
|
||||||
*args: Any,
|
|
||||||
feed_options: dict[str, Any] | None = None,
|
|
||||||
preargs: Iterable[Any] = (),
|
|
||||||
**kwargs: Any,
|
|
||||||
) -> _StorageT:
|
|
||||||
warnings.warn(
|
|
||||||
"scrapy.extensions.feedexport.build_storage() is deprecated, call the builder directly.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
kwargs["feed_options"] = feed_options
|
|
||||||
return builder(*preargs, uri, *args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
class ItemFilter:
|
class ItemFilter:
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,6 @@ See documentation in docs/topics/email.rst
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import warnings
|
|
||||||
from email import encoders as Encoders
|
from email import encoders as Encoders
|
||||||
from email.mime.base import MIMEBase
|
from email.mime.base import MIMEBase
|
||||||
from email.mime.multipart import MIMEMultipart
|
from email.mime.multipart import MIMEMultipart
|
||||||
|
|
@ -20,7 +19,6 @@ from typing import IO, TYPE_CHECKING, Any
|
||||||
from twisted.internet import ssl
|
from twisted.internet import ssl
|
||||||
from twisted.internet.defer import Deferred
|
from twisted.internet.defer import Deferred
|
||||||
|
|
||||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
|
||||||
from scrapy.utils.misc import arg_to_iter
|
from scrapy.utils.misc import arg_to_iter
|
||||||
from scrapy.utils.python import to_bytes
|
from scrapy.utils.python import to_bytes
|
||||||
|
|
||||||
|
|
@ -35,7 +33,6 @@ if TYPE_CHECKING:
|
||||||
from typing_extensions import Self
|
from typing_extensions import Self
|
||||||
|
|
||||||
from scrapy.crawler import Crawler
|
from scrapy.crawler import Crawler
|
||||||
from scrapy.settings import BaseSettings
|
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -73,21 +70,9 @@ class MailSender:
|
||||||
self.mailfrom: str = mailfrom
|
self.mailfrom: str = mailfrom
|
||||||
self.debug: bool = debug
|
self.debug: bool = debug
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_settings(cls, settings: BaseSettings) -> Self:
|
|
||||||
warnings.warn(
|
|
||||||
f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
return cls._from_settings(settings)
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||||
return cls._from_settings(crawler.settings)
|
settings = crawler.settings
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _from_settings(cls, settings: BaseSettings) -> Self:
|
|
||||||
return cls(
|
return cls(
|
||||||
smtphost=settings["MAIL_HOST"],
|
smtphost=settings["MAIL_HOST"],
|
||||||
mailfrom=settings["MAIL_FROM"],
|
mailfrom=settings["MAIL_FROM"],
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,7 @@ if TYPE_CHECKING:
|
||||||
|
|
||||||
from scrapy import Spider
|
from scrapy import Spider
|
||||||
from scrapy.crawler import Crawler
|
from scrapy.crawler import Crawler
|
||||||
from scrapy.settings import BaseSettings, Settings
|
from scrapy.settings import Settings
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -100,43 +100,15 @@ class MiddlewareManager(ABC):
|
||||||
def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]:
|
def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _build_from_settings(objcls: type[_T], settings: BaseSettings) -> _T:
|
|
||||||
if hasattr(objcls, "from_settings"):
|
|
||||||
instance = objcls.from_settings(settings) # type: ignore[attr-defined]
|
|
||||||
method_name = "from_settings"
|
|
||||||
else:
|
|
||||||
instance = objcls()
|
|
||||||
method_name = "__new__"
|
|
||||||
if instance is None:
|
|
||||||
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
|
|
||||||
return cast("_T", instance)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self:
|
|
||||||
warnings.warn(
|
|
||||||
f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
return cls._from_settings(settings, crawler)
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||||
return cls._from_settings(crawler.settings, crawler)
|
mwlist = cls._get_mwlist_from_settings(crawler.settings)
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self:
|
|
||||||
mwlist = cls._get_mwlist_from_settings(settings)
|
|
||||||
middlewares = []
|
middlewares = []
|
||||||
enabled = []
|
enabled = []
|
||||||
for clspath in mwlist:
|
for clspath in mwlist:
|
||||||
try:
|
try:
|
||||||
mwcls = load_object(clspath)
|
mwcls = load_object(clspath)
|
||||||
if crawler is not None:
|
mw = build_from_crawler(mwcls, crawler)
|
||||||
mw = build_from_crawler(mwcls, crawler)
|
|
||||||
else:
|
|
||||||
mw = MiddlewareManager._build_from_settings(mwcls, settings)
|
|
||||||
middlewares.append(mw)
|
middlewares.append(mw)
|
||||||
enabled.append(clspath)
|
enabled.append(clspath)
|
||||||
except NotConfigured as e:
|
except NotConfigured as e:
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,6 @@ import hashlib
|
||||||
import logging
|
import logging
|
||||||
import mimetypes
|
import mimetypes
|
||||||
import time
|
import time
|
||||||
import warnings
|
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from ftplib import FTP
|
from ftplib import FTP
|
||||||
|
|
@ -25,17 +24,15 @@ from itemadapter import ItemAdapter
|
||||||
from twisted.internet.defer import Deferred, maybeDeferred
|
from twisted.internet.defer import Deferred, maybeDeferred
|
||||||
from twisted.internet.threads import deferToThread
|
from twisted.internet.threads import deferToThread
|
||||||
|
|
||||||
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
|
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||||
from scrapy.http import Request, Response
|
from scrapy.http import Request, Response
|
||||||
from scrapy.http.request import NO_CALLBACK
|
from scrapy.http.request import NO_CALLBACK
|
||||||
from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline
|
from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline
|
||||||
from scrapy.settings import BaseSettings, Settings
|
|
||||||
from scrapy.utils.boto import is_botocore_available
|
from scrapy.utils.boto import is_botocore_available
|
||||||
from scrapy.utils.datatypes import CaseInsensitiveDict
|
from scrapy.utils.datatypes import CaseInsensitiveDict
|
||||||
from scrapy.utils.deprecate import method_is_overridden
|
|
||||||
from scrapy.utils.ftp import ftp_store_file
|
from scrapy.utils.ftp import ftp_store_file
|
||||||
from scrapy.utils.log import failure_to_exc_info
|
from scrapy.utils.log import failure_to_exc_info
|
||||||
from scrapy.utils.python import get_func_args, global_object_name, to_bytes
|
from scrapy.utils.python import to_bytes
|
||||||
from scrapy.utils.request import referer_str
|
from scrapy.utils.request import referer_str
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
|
@ -49,6 +46,7 @@ if TYPE_CHECKING:
|
||||||
|
|
||||||
from scrapy import Spider
|
from scrapy import Spider
|
||||||
from scrapy.crawler import Crawler
|
from scrapy.crawler import Crawler
|
||||||
|
from scrapy.settings import BaseSettings
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -161,7 +159,7 @@ class S3FilesStore:
|
||||||
AWS_USE_SSL = None
|
AWS_USE_SSL = None
|
||||||
AWS_VERIFY = None
|
AWS_VERIFY = None
|
||||||
|
|
||||||
POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
|
POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler()
|
||||||
HEADERS = {
|
HEADERS = {
|
||||||
"Cache-Control": "max-age=172800",
|
"Cache-Control": "max-age=172800",
|
||||||
}
|
}
|
||||||
|
|
@ -280,7 +278,7 @@ class GCSFilesStore:
|
||||||
CACHE_CONTROL = "max-age=172800"
|
CACHE_CONTROL = "max-age=172800"
|
||||||
|
|
||||||
# The bucket's default object ACL will be applied to the object.
|
# The bucket's default object ACL will be applied to the object.
|
||||||
# Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
|
# Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_crawler().
|
||||||
POLICY = None
|
POLICY = None
|
||||||
|
|
||||||
def __init__(self, uri: str):
|
def __init__(self, uri: str):
|
||||||
|
|
@ -446,9 +444,8 @@ class FilesPipeline(MediaPipeline):
|
||||||
self,
|
self,
|
||||||
store_uri: str | PathLike[str],
|
store_uri: str | PathLike[str],
|
||||||
download_func: Callable[[Request, Spider], Response] | None = None,
|
download_func: Callable[[Request, Spider], Response] | None = None,
|
||||||
settings: Settings | dict[str, Any] | None = None,
|
|
||||||
*,
|
*,
|
||||||
crawler: Crawler | None = None,
|
crawler: Crawler,
|
||||||
):
|
):
|
||||||
if not (store_uri and (store_uri := _to_string(store_uri))):
|
if not (store_uri and (store_uri := _to_string(store_uri))):
|
||||||
from scrapy.pipelines.images import ImagesPipeline # noqa: PLC0415
|
from scrapy.pipelines.images import ImagesPipeline # noqa: PLC0415
|
||||||
|
|
@ -461,18 +458,7 @@ class FilesPipeline(MediaPipeline):
|
||||||
f"to enable {self.__class__.__name__}."
|
f"to enable {self.__class__.__name__}."
|
||||||
)
|
)
|
||||||
|
|
||||||
if crawler is not None:
|
settings = crawler.settings
|
||||||
if settings is not None:
|
|
||||||
warnings.warn(
|
|
||||||
f"FilesPipeline.__init__() was called with a crawler instance and a settings instance"
|
|
||||||
f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored"
|
|
||||||
f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
settings = crawler.settings
|
|
||||||
elif isinstance(settings, dict) or settings is None:
|
|
||||||
settings = Settings(settings)
|
|
||||||
cls_name = "FilesPipeline"
|
cls_name = "FilesPipeline"
|
||||||
self.store: FilesStoreProtocol = self._get_store(store_uri)
|
self.store: FilesStoreProtocol = self._get_store(store_uri)
|
||||||
resolve = functools.partial(
|
resolve = functools.partial(
|
||||||
|
|
@ -490,51 +476,14 @@ class FilesPipeline(MediaPipeline):
|
||||||
resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD
|
resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD
|
||||||
)
|
)
|
||||||
|
|
||||||
super().__init__(
|
super().__init__(download_func=download_func, crawler=crawler)
|
||||||
download_func=download_func,
|
|
||||||
settings=settings if not crawler else None,
|
|
||||||
crawler=crawler,
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_settings(cls, settings: Settings) -> Self:
|
|
||||||
warnings.warn(
|
|
||||||
f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
return cls._from_settings(settings, None)
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||||
if method_is_overridden(cls, FilesPipeline, "from_settings"):
|
settings = crawler.settings
|
||||||
warnings.warn(
|
|
||||||
f"{global_object_name(cls)} overrides FilesPipeline.from_settings()."
|
|
||||||
f" This method is deprecated and won't be called in future Scrapy versions,"
|
|
||||||
f" please update your code so that it overrides from_crawler() instead.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
)
|
|
||||||
o = cls.from_settings(crawler.settings)
|
|
||||||
o._finish_init(crawler)
|
|
||||||
return o
|
|
||||||
return cls._from_settings(crawler.settings, crawler)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self:
|
|
||||||
cls._update_stores(settings)
|
cls._update_stores(settings)
|
||||||
store_uri = settings["FILES_STORE"]
|
store_uri = settings["FILES_STORE"]
|
||||||
if "crawler" in get_func_args(cls.__init__):
|
return cls(store_uri, crawler=crawler)
|
||||||
o = cls(store_uri, crawler=crawler)
|
|
||||||
else:
|
|
||||||
o = cls(store_uri, settings=settings)
|
|
||||||
if crawler:
|
|
||||||
o._finish_init(crawler)
|
|
||||||
warnings.warn(
|
|
||||||
f"{global_object_name(cls)}.__init__() doesn't take a crawler argument."
|
|
||||||
" This is deprecated and the argument will be required in future Scrapy versions.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
)
|
|
||||||
return o
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _update_stores(cls, settings: BaseSettings) -> None:
|
def _update_stores(cls, settings: BaseSettings) -> None:
|
||||||
|
|
|
||||||
|
|
@ -8,19 +8,17 @@ from __future__ import annotations
|
||||||
|
|
||||||
import functools
|
import functools
|
||||||
import hashlib
|
import hashlib
|
||||||
import warnings
|
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
from itemadapter import ItemAdapter
|
from itemadapter import ItemAdapter
|
||||||
|
|
||||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
from scrapy.exceptions import NotConfigured
|
||||||
from scrapy.http import Request, Response
|
from scrapy.http import Request, Response
|
||||||
from scrapy.http.request import NO_CALLBACK
|
from scrapy.http.request import NO_CALLBACK
|
||||||
from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum
|
from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum
|
||||||
from scrapy.settings import Settings
|
from scrapy.utils.python import to_bytes
|
||||||
from scrapy.utils.python import get_func_args, global_object_name, to_bytes
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import Callable, Iterable
|
from collections.abc import Callable, Iterable
|
||||||
|
|
@ -58,9 +56,8 @@ class ImagesPipeline(FilesPipeline):
|
||||||
self,
|
self,
|
||||||
store_uri: str | PathLike[str],
|
store_uri: str | PathLike[str],
|
||||||
download_func: Callable[[Request, Spider], Response] | None = None,
|
download_func: Callable[[Request, Spider], Response] | None = None,
|
||||||
settings: Settings | dict[str, Any] | None = None,
|
|
||||||
*,
|
*,
|
||||||
crawler: Crawler | None = None,
|
crawler: Crawler,
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
from PIL import Image, ImageOps # noqa: PLC0415
|
from PIL import Image, ImageOps # noqa: PLC0415
|
||||||
|
|
@ -72,26 +69,9 @@ class ImagesPipeline(FilesPipeline):
|
||||||
"ImagesPipeline requires installing Pillow 8.3.2 or later"
|
"ImagesPipeline requires installing Pillow 8.3.2 or later"
|
||||||
)
|
)
|
||||||
|
|
||||||
super().__init__(
|
super().__init__(store_uri, download_func=download_func, crawler=crawler)
|
||||||
store_uri,
|
|
||||||
settings=settings if not crawler else None,
|
|
||||||
download_func=download_func,
|
|
||||||
crawler=crawler,
|
|
||||||
)
|
|
||||||
|
|
||||||
if crawler is not None:
|
|
||||||
if settings is not None:
|
|
||||||
warnings.warn(
|
|
||||||
f"ImagesPipeline.__init__() was called with a crawler instance and a settings instance"
|
|
||||||
f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored"
|
|
||||||
f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
settings = crawler.settings
|
|
||||||
elif isinstance(settings, dict) or settings is None:
|
|
||||||
settings = Settings(settings)
|
|
||||||
|
|
||||||
|
settings = crawler.settings
|
||||||
resolve = functools.partial(
|
resolve = functools.partial(
|
||||||
self._key_for_pipe,
|
self._key_for_pipe,
|
||||||
base_class_name="ImagesPipeline",
|
base_class_name="ImagesPipeline",
|
||||||
|
|
@ -121,21 +101,11 @@ class ImagesPipeline(FilesPipeline):
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self:
|
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||||
|
settings = crawler.settings
|
||||||
cls._update_stores(settings)
|
cls._update_stores(settings)
|
||||||
store_uri = settings["IMAGES_STORE"]
|
store_uri = settings["IMAGES_STORE"]
|
||||||
if "crawler" in get_func_args(cls.__init__):
|
return cls(store_uri, crawler=crawler)
|
||||||
o = cls(store_uri, crawler=crawler)
|
|
||||||
else:
|
|
||||||
o = cls(store_uri, settings=settings)
|
|
||||||
if crawler:
|
|
||||||
o._finish_init(crawler)
|
|
||||||
warnings.warn(
|
|
||||||
f"{global_object_name(cls)}.__init__() doesn't take a crawler argument."
|
|
||||||
" This is deprecated and the argument will be required in future Scrapy versions.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
)
|
|
||||||
return o
|
|
||||||
|
|
||||||
def file_downloaded(
|
def file_downloaded(
|
||||||
self,
|
self,
|
||||||
|
|
|
||||||
|
|
@ -19,14 +19,13 @@ from twisted.python.versions import Version
|
||||||
|
|
||||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||||
from scrapy.http.request import NO_CALLBACK, Request
|
from scrapy.http.request import NO_CALLBACK, Request
|
||||||
from scrapy.settings import Settings
|
|
||||||
from scrapy.utils.asyncio import call_later
|
from scrapy.utils.asyncio import call_later
|
||||||
from scrapy.utils.datatypes import SequenceExclude
|
from scrapy.utils.datatypes import SequenceExclude
|
||||||
from scrapy.utils.decorators import _warn_spider_arg
|
from scrapy.utils.decorators import _warn_spider_arg
|
||||||
from scrapy.utils.defer import _DEFER_DELAY, _defer_sleep, deferred_from_coro
|
from scrapy.utils.defer import _DEFER_DELAY, _defer_sleep, deferred_from_coro
|
||||||
from scrapy.utils.log import failure_to_exc_info
|
from scrapy.utils.log import failure_to_exc_info
|
||||||
from scrapy.utils.misc import arg_to_iter
|
from scrapy.utils.misc import arg_to_iter
|
||||||
from scrapy.utils.python import get_func_args, global_object_name
|
from scrapy.utils.python import global_object_name
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import Callable, Generator
|
from collections.abc import Callable, Generator
|
||||||
|
|
@ -37,6 +36,7 @@ if TYPE_CHECKING:
|
||||||
from scrapy import Spider
|
from scrapy import Spider
|
||||||
from scrapy.crawler import Crawler
|
from scrapy.crawler import Crawler
|
||||||
from scrapy.http import Response
|
from scrapy.http import Response
|
||||||
|
from scrapy.settings import Settings
|
||||||
from scrapy.utils.request import RequestFingerprinterProtocol
|
from scrapy.utils.request import RequestFingerprinterProtocol
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -55,10 +55,6 @@ logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class MediaPipeline(ABC):
|
class MediaPipeline(ABC):
|
||||||
crawler: Crawler
|
|
||||||
_fingerprinter: RequestFingerprinterProtocol
|
|
||||||
_modern_init = False
|
|
||||||
|
|
||||||
LOG_FAILED_RESULTS: bool = True
|
LOG_FAILED_RESULTS: bool = True
|
||||||
|
|
||||||
class SpiderInfo:
|
class SpiderInfo:
|
||||||
|
|
@ -73,24 +69,17 @@ class MediaPipeline(ABC):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
download_func: Callable[[Request, Spider], Response] | None = None,
|
download_func: Callable[[Request, Spider], Response] | None = None,
|
||||||
settings: Settings | dict[str, Any] | None = None,
|
|
||||||
*,
|
*,
|
||||||
crawler: Crawler | None = None,
|
crawler: Crawler,
|
||||||
):
|
):
|
||||||
|
self.crawler: Crawler = crawler
|
||||||
|
assert crawler.request_fingerprinter
|
||||||
|
self._fingerprinter: RequestFingerprinterProtocol = (
|
||||||
|
crawler.request_fingerprinter
|
||||||
|
)
|
||||||
self.download_func = download_func
|
self.download_func = download_func
|
||||||
|
|
||||||
if crawler is not None:
|
settings = crawler.settings
|
||||||
if settings is not None:
|
|
||||||
warnings.warn(
|
|
||||||
f"MediaPipeline.__init__() was called with a crawler instance and a settings instance"
|
|
||||||
f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored"
|
|
||||||
f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
settings = crawler.settings
|
|
||||||
elif isinstance(settings, dict) or settings is None:
|
|
||||||
settings = Settings(settings)
|
|
||||||
resolve = functools.partial(
|
resolve = functools.partial(
|
||||||
self._key_for_pipe, base_class_name="MediaPipeline", settings=settings
|
self._key_for_pipe, base_class_name="MediaPipeline", settings=settings
|
||||||
)
|
)
|
||||||
|
|
@ -99,27 +88,6 @@ class MediaPipeline(ABC):
|
||||||
)
|
)
|
||||||
self._handle_statuses(self.allow_redirects)
|
self._handle_statuses(self.allow_redirects)
|
||||||
|
|
||||||
if crawler:
|
|
||||||
self._finish_init(crawler)
|
|
||||||
self._modern_init = True
|
|
||||||
else:
|
|
||||||
warnings.warn(
|
|
||||||
f"MediaPipeline.__init__() was called without the crawler argument"
|
|
||||||
f" when creating {global_object_name(self.__class__)}."
|
|
||||||
f" This is deprecated and the argument will be required in future Scrapy versions.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _finish_init(self, crawler: Crawler) -> None:
|
|
||||||
# This was done in from_crawler() before 2.12, now it's done in __init__()
|
|
||||||
# if the crawler was passed to it and may be needed to be called in other
|
|
||||||
# deprecated code paths explicitly too. After the crawler argument of __init__()
|
|
||||||
# becomes mandatory this should be inlined there.
|
|
||||||
self.crawler = crawler
|
|
||||||
assert crawler.request_fingerprinter
|
|
||||||
self._fingerprinter = crawler.request_fingerprinter
|
|
||||||
|
|
||||||
def _handle_statuses(self, allow_redirects: bool) -> None:
|
def _handle_statuses(self, allow_redirects: bool) -> None:
|
||||||
self.handle_httpstatus_list = None
|
self.handle_httpstatus_list = None
|
||||||
if allow_redirects:
|
if allow_redirects:
|
||||||
|
|
@ -143,30 +111,7 @@ class MediaPipeline(ABC):
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||||
pipe: Self
|
return cls(crawler=crawler)
|
||||||
if hasattr(cls, "from_settings"):
|
|
||||||
pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined]
|
|
||||||
warnings.warn(
|
|
||||||
f"{global_object_name(cls)} has from_settings() and either doesn't have"
|
|
||||||
" from_crawler() or calls MediaPipeline.from_crawler() from it,"
|
|
||||||
" so from_settings() was used to create the instance of it."
|
|
||||||
" This is deprecated and calling from_settings() will be removed"
|
|
||||||
" in a future Scrapy version. Please move the initialization code into"
|
|
||||||
" from_crawler() or __init__().",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
)
|
|
||||||
elif "crawler" in get_func_args(cls.__init__):
|
|
||||||
pipe = cls(crawler=crawler)
|
|
||||||
else:
|
|
||||||
pipe = cls()
|
|
||||||
warnings.warn(
|
|
||||||
f"{global_object_name(cls)}.__init__() doesn't take a crawler argument."
|
|
||||||
" This is deprecated and the argument will be required in future Scrapy versions.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
)
|
|
||||||
if not pipe._modern_init:
|
|
||||||
pipe._finish_init(crawler)
|
|
||||||
return pipe
|
|
||||||
|
|
||||||
@_warn_spider_arg
|
@_warn_spider_arg
|
||||||
def open_spider(self, spider: Spider | None = None) -> None:
|
def open_spider(self, spider: Spider | None = None) -> None:
|
||||||
|
|
|
||||||
|
|
@ -152,7 +152,6 @@ __all__ = [
|
||||||
"REFERER_ENABLED",
|
"REFERER_ENABLED",
|
||||||
"REFERRER_POLICY",
|
"REFERRER_POLICY",
|
||||||
"REQUEST_FINGERPRINTER_CLASS",
|
"REQUEST_FINGERPRINTER_CLASS",
|
||||||
"REQUEST_FINGERPRINTER_IMPLEMENTATION",
|
|
||||||
"RETRY_ENABLED",
|
"RETRY_ENABLED",
|
||||||
"RETRY_EXCEPTIONS",
|
"RETRY_EXCEPTIONS",
|
||||||
"RETRY_HTTP_CODES",
|
"RETRY_HTTP_CODES",
|
||||||
|
|
@ -451,7 +450,6 @@ REFERER_ENABLED = True
|
||||||
REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
|
REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
|
||||||
|
|
||||||
REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
|
REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
|
||||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL"
|
|
||||||
|
|
||||||
RETRY_ENABLED = True
|
RETRY_ENABLED = True
|
||||||
RETRY_EXCEPTIONS = [
|
RETRY_EXCEPTIONS = [
|
||||||
|
|
|
||||||
|
|
@ -317,31 +317,6 @@ def process_chain(
|
||||||
return d
|
return d
|
||||||
|
|
||||||
|
|
||||||
def process_chain_both(
|
|
||||||
callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
|
|
||||||
errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]],
|
|
||||||
input: Any, # noqa: A002
|
|
||||||
*a: _P.args,
|
|
||||||
**kw: _P.kwargs,
|
|
||||||
) -> Deferred:
|
|
||||||
"""Return a Deferred built by chaining the given callbacks and errbacks"""
|
|
||||||
warnings.warn(
|
|
||||||
"process_chain_both() is deprecated and will be removed in a future"
|
|
||||||
" Scrapy version.",
|
|
||||||
ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
d: Deferred = Deferred()
|
|
||||||
for cb, eb in zip(callbacks, errbacks, strict=False):
|
|
||||||
d.addCallback(cb, *a, **kw)
|
|
||||||
d.addErrback(eb, *a, **kw)
|
|
||||||
if isinstance(input, failure.Failure):
|
|
||||||
d.errback(input)
|
|
||||||
else:
|
|
||||||
d.callback(input)
|
|
||||||
return d
|
|
||||||
|
|
||||||
|
|
||||||
def process_parallel(
|
def process_parallel(
|
||||||
callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]],
|
callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]],
|
||||||
input: _T, # noqa: A002
|
input: _T, # noqa: A002
|
||||||
|
|
|
||||||
|
|
@ -129,52 +129,10 @@ def rel_has_nofollow(rel: str | None) -> bool:
|
||||||
return rel is not None and "nofollow" in rel.replace(",", " ").split()
|
return rel is not None and "nofollow" in rel.replace(",", " ").split()
|
||||||
|
|
||||||
|
|
||||||
def create_instance(objcls, settings, crawler, *args, **kwargs):
|
|
||||||
"""Construct a class instance using its ``from_crawler`` or
|
|
||||||
``from_settings`` constructors, if available.
|
|
||||||
|
|
||||||
At least one of ``settings`` and ``crawler`` needs to be different from
|
|
||||||
``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
|
|
||||||
If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
|
|
||||||
tried.
|
|
||||||
|
|
||||||
``*args`` and ``**kwargs`` are forwarded to the constructors.
|
|
||||||
|
|
||||||
Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
|
|
||||||
|
|
||||||
.. versionchanged:: 2.2
|
|
||||||
Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
|
|
||||||
extension has not been implemented correctly).
|
|
||||||
"""
|
|
||||||
warnings.warn(
|
|
||||||
"The create_instance() function is deprecated. "
|
|
||||||
"Please use build_from_crawler() instead.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
|
|
||||||
if settings is None:
|
|
||||||
if crawler is None:
|
|
||||||
raise ValueError("Specify at least one of settings and crawler.")
|
|
||||||
settings = crawler.settings
|
|
||||||
if crawler and hasattr(objcls, "from_crawler"):
|
|
||||||
instance = objcls.from_crawler(crawler, *args, **kwargs)
|
|
||||||
method_name = "from_crawler"
|
|
||||||
elif hasattr(objcls, "from_settings"):
|
|
||||||
instance = objcls.from_settings(settings, *args, **kwargs)
|
|
||||||
method_name = "from_settings"
|
|
||||||
else:
|
|
||||||
instance = objcls(*args, **kwargs)
|
|
||||||
method_name = "__new__"
|
|
||||||
if instance is None:
|
|
||||||
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
|
|
||||||
return instance
|
|
||||||
|
|
||||||
|
|
||||||
def build_from_crawler(
|
def build_from_crawler(
|
||||||
objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any
|
objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any
|
||||||
) -> T:
|
) -> T:
|
||||||
"""Construct a class instance using its ``from_crawler`` or ``from_settings`` constructor.
|
"""Construct a class instance using its ``from_crawler()`` or ``__init__()`` constructor.
|
||||||
|
|
||||||
.. versionadded:: 2.12
|
.. versionadded:: 2.12
|
||||||
|
|
||||||
|
|
@ -185,17 +143,6 @@ def build_from_crawler(
|
||||||
if hasattr(objcls, "from_crawler"):
|
if hasattr(objcls, "from_crawler"):
|
||||||
instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined]
|
instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined]
|
||||||
method_name = "from_crawler"
|
method_name = "from_crawler"
|
||||||
elif hasattr(objcls, "from_settings"):
|
|
||||||
warnings.warn(
|
|
||||||
f"{objcls.__qualname__} has from_settings() but not from_crawler()."
|
|
||||||
" This is deprecated and calling from_settings() will be removed in a future"
|
|
||||||
" Scrapy version. You can implement a simple from_crawler() that calls"
|
|
||||||
" from_settings() with crawler.settings.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
instance = objcls.from_settings(crawler.settings, *args, **kwargs) # type: ignore[attr-defined]
|
|
||||||
method_name = "from_settings"
|
|
||||||
else:
|
else:
|
||||||
instance = objcls(*args, **kwargs)
|
instance = objcls(*args, **kwargs)
|
||||||
method_name = "__new__"
|
method_name = "__new__"
|
||||||
|
|
|
||||||
|
|
@ -8,14 +8,12 @@ import gc
|
||||||
import inspect
|
import inspect
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
import warnings
|
|
||||||
import weakref
|
import weakref
|
||||||
from collections.abc import AsyncIterator, Iterable, Mapping
|
from collections.abc import AsyncIterator, Iterable, Mapping
|
||||||
from functools import partial, wraps
|
from functools import partial, wraps
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload
|
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload
|
||||||
|
|
||||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
|
||||||
from scrapy.utils.asyncgen import as_async_generator
|
from scrapy.utils.asyncgen import as_async_generator
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
|
@ -32,47 +30,6 @@ _VT = TypeVar("_VT")
|
||||||
_P = ParamSpec("_P")
|
_P = ParamSpec("_P")
|
||||||
|
|
||||||
|
|
||||||
def flatten(x: Iterable[Any]) -> list[Any]:
|
|
||||||
"""flatten(sequence) -> list
|
|
||||||
|
|
||||||
Returns a single, flat list which contains all elements retrieved
|
|
||||||
from the sequence and all recursively contained sub-sequences
|
|
||||||
(iterables).
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
>>> [1, 2, [3,4], (5,6)]
|
|
||||||
[1, 2, [3, 4], (5, 6)]
|
|
||||||
>>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)])
|
|
||||||
[1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]
|
|
||||||
>>> flatten(["foo", "bar"])
|
|
||||||
['foo', 'bar']
|
|
||||||
>>> flatten(["foo", ["baz", 42], "bar"])
|
|
||||||
['foo', 'baz', 42, 'bar']
|
|
||||||
"""
|
|
||||||
warnings.warn(
|
|
||||||
"The flatten function is deprecated and will be removed in a future version of Scrapy.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
return list(iflatten(x))
|
|
||||||
|
|
||||||
|
|
||||||
def iflatten(x: Iterable[Any]) -> Iterable[Any]:
|
|
||||||
"""iflatten(sequence) -> iterator
|
|
||||||
|
|
||||||
Similar to ``.flatten()``, but returns iterator instead"""
|
|
||||||
warnings.warn(
|
|
||||||
"The iflatten function is deprecated and will be removed in a future version of Scrapy.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
for el in x:
|
|
||||||
if is_listlike(el):
|
|
||||||
yield from iflatten(el)
|
|
||||||
else:
|
|
||||||
yield el
|
|
||||||
|
|
||||||
|
|
||||||
def is_listlike(x: Any) -> bool:
|
def is_listlike(x: Any) -> bool:
|
||||||
"""
|
"""
|
||||||
>>> is_listlike("foo")
|
>>> is_listlike("foo")
|
||||||
|
|
@ -289,31 +246,6 @@ def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]:
|
||||||
return args, kwargs
|
return args, kwargs
|
||||||
|
|
||||||
|
|
||||||
def equal_attributes(
|
|
||||||
obj1: Any, obj2: Any, attributes: list[str | Callable[[Any], Any]] | None
|
|
||||||
) -> bool:
|
|
||||||
"""Compare two objects attributes"""
|
|
||||||
warnings.warn(
|
|
||||||
"The equal_attributes function is deprecated and will be removed in a future version of Scrapy.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
# not attributes given return False by default
|
|
||||||
if not attributes:
|
|
||||||
return False
|
|
||||||
|
|
||||||
temp1, temp2 = object(), object()
|
|
||||||
for attr in attributes:
|
|
||||||
# support callables like itemgetter
|
|
||||||
if callable(attr):
|
|
||||||
if attr(obj1) != attr(obj2):
|
|
||||||
return False
|
|
||||||
elif getattr(obj1, attr, temp1) != getattr(obj2, attr, temp2):
|
|
||||||
return False
|
|
||||||
# all attributes equal
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def without_none_values(iterable: Mapping[_KT, _VT]) -> dict[_KT, _VT]: ...
|
def without_none_values(iterable: Mapping[_KT, _VT]) -> dict[_KT, _VT]: ...
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,16 +7,13 @@ from __future__ import annotations
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import warnings
|
|
||||||
from typing import TYPE_CHECKING, Any, Protocol
|
from typing import TYPE_CHECKING, Any, Protocol
|
||||||
from urllib.parse import urlunparse
|
from urllib.parse import urlunparse
|
||||||
from weakref import WeakKeyDictionary
|
from weakref import WeakKeyDictionary
|
||||||
|
|
||||||
from w3lib.http import basic_auth_header
|
|
||||||
from w3lib.url import canonicalize_url
|
from w3lib.url import canonicalize_url
|
||||||
|
|
||||||
from scrapy import Request, Spider
|
from scrapy import Request, Spider
|
||||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
|
||||||
from scrapy.utils.httpobj import urlparse_cached
|
from scrapy.utils.httpobj import urlparse_cached
|
||||||
from scrapy.utils.misc import load_object
|
from scrapy.utils.misc import load_object
|
||||||
from scrapy.utils.python import to_bytes, to_unicode
|
from scrapy.utils.python import to_bytes, to_unicode
|
||||||
|
|
@ -120,41 +117,12 @@ class RequestFingerprinter:
|
||||||
return cls(crawler)
|
return cls(crawler)
|
||||||
|
|
||||||
def __init__(self, crawler: Crawler | None = None):
|
def __init__(self, crawler: Crawler | None = None):
|
||||||
if crawler:
|
|
||||||
implementation = crawler.settings.get(
|
|
||||||
"REQUEST_FINGERPRINTER_IMPLEMENTATION"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
implementation = "SENTINEL"
|
|
||||||
|
|
||||||
if implementation != "SENTINEL":
|
|
||||||
message = (
|
|
||||||
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"
|
|
||||||
"It will be removed in a future version of Scrapy."
|
|
||||||
)
|
|
||||||
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
|
|
||||||
self._fingerprint = fingerprint
|
self._fingerprint = fingerprint
|
||||||
|
|
||||||
def fingerprint(self, request: Request) -> bytes:
|
def fingerprint(self, request: Request) -> bytes:
|
||||||
return self._fingerprint(request)
|
return self._fingerprint(request)
|
||||||
|
|
||||||
|
|
||||||
def request_authenticate(
|
|
||||||
request: Request,
|
|
||||||
username: str,
|
|
||||||
password: str,
|
|
||||||
) -> None:
|
|
||||||
"""Authenticate the given request (in place) using the HTTP basic access
|
|
||||||
authentication mechanism (RFC 2617) and the given username and password
|
|
||||||
"""
|
|
||||||
warnings.warn(
|
|
||||||
"The request_authenticate function is deprecated and will be removed in a future version of Scrapy.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
request.headers["Authorization"] = basic_auth_header(username, password)
|
|
||||||
|
|
||||||
|
|
||||||
def request_httprepr(request: Request) -> bytes:
|
def request_httprepr(request: Request) -> bytes:
|
||||||
"""Return the raw HTTP representation (as bytes) of the given request.
|
"""Return the raw HTTP representation (as bytes) of the given request.
|
||||||
This is provided only for reference since it's not the actual stream of
|
This is provided only for reference since it's not the actual stream of
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,11 @@
|
||||||
import datetime
|
import datetime
|
||||||
import decimal
|
import decimal
|
||||||
import json
|
import json
|
||||||
import warnings
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from itemadapter import ItemAdapter, is_item
|
from itemadapter import ItemAdapter, is_item
|
||||||
from twisted.internet import defer
|
from twisted.internet import defer
|
||||||
|
|
||||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
|
||||||
from scrapy.http import Request, Response
|
from scrapy.http import Request, Response
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -35,13 +33,3 @@ class ScrapyJSONEncoder(json.JSONEncoder):
|
||||||
if is_item(o):
|
if is_item(o):
|
||||||
return ItemAdapter(o).asdict()
|
return ItemAdapter(o).asdict()
|
||||||
return super().default(o)
|
return super().default(o)
|
||||||
|
|
||||||
|
|
||||||
class ScrapyJSONDecoder(json.JSONDecoder):
|
|
||||||
def __init__(self, *args, **kwargs):
|
|
||||||
warnings.warn(
|
|
||||||
"The ScrapyJSONDecoder class is deprecated and will be removed in a future version of Scrapy.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
super().__init__(*args, **kwargs)
|
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ from importlib import import_module
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from posixpath import split
|
from posixpath import split
|
||||||
from typing import TYPE_CHECKING, Any, TypeVar, cast
|
from typing import TYPE_CHECKING, Any, TypeVar, cast
|
||||||
from unittest import TestCase, mock
|
from unittest import mock
|
||||||
|
|
||||||
from twisted.trial.unittest import SkipTest
|
from twisted.trial.unittest import SkipTest
|
||||||
from twisted.web.client import Agent
|
from twisted.web.client import Agent
|
||||||
|
|
@ -166,20 +166,6 @@ def get_testenv() -> dict[str, str]:
|
||||||
return env
|
return env
|
||||||
|
|
||||||
|
|
||||||
def assert_samelines(
|
|
||||||
testcase: TestCase, text1: str, text2: str, msg: str | None = None
|
|
||||||
) -> None:
|
|
||||||
"""Asserts text1 and text2 have the same lines, ignoring differences in
|
|
||||||
line endings between platforms
|
|
||||||
"""
|
|
||||||
warnings.warn(
|
|
||||||
"The assert_samelines function is deprecated and will be removed in a future version of Scrapy.",
|
|
||||||
category=ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) # noqa: PT009
|
|
||||||
|
|
||||||
|
|
||||||
def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:
|
def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:
|
||||||
q: asyncio.Queue[_T] = asyncio.Queue()
|
q: asyncio.Queue[_T] = asyncio.Queue()
|
||||||
getter = q.get()
|
getter = q.get()
|
||||||
|
|
|
||||||
|
|
@ -756,37 +756,6 @@ class TestBuildFromCrawler:
|
||||||
assert len(w) == 0
|
assert len(w) == 0
|
||||||
assert pipe.store
|
assert pipe.store
|
||||||
|
|
||||||
def test_has_old_init(self):
|
|
||||||
class Pipeline(FilesPipeline):
|
|
||||||
def __init__(self, store_uri, download_func=None, settings=None):
|
|
||||||
super().__init__(store_uri, download_func, settings)
|
|
||||||
self._init_called = True
|
|
||||||
|
|
||||||
with warnings.catch_warnings(record=True) as w:
|
|
||||||
pipe = Pipeline.from_crawler(self.crawler)
|
|
||||||
assert pipe.crawler == self.crawler
|
|
||||||
assert pipe._fingerprinter
|
|
||||||
assert len(w) == 2
|
|
||||||
assert pipe._init_called
|
|
||||||
|
|
||||||
def test_has_from_settings(self):
|
|
||||||
class Pipeline(FilesPipeline):
|
|
||||||
_from_settings_called = False
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_settings(cls, settings):
|
|
||||||
o = super().from_settings(settings)
|
|
||||||
o._from_settings_called = True
|
|
||||||
return o
|
|
||||||
|
|
||||||
with warnings.catch_warnings(record=True) as w:
|
|
||||||
pipe = Pipeline.from_crawler(self.crawler)
|
|
||||||
assert pipe.crawler == self.crawler
|
|
||||||
assert pipe._fingerprinter
|
|
||||||
assert len(w) == 3
|
|
||||||
assert pipe.store
|
|
||||||
assert pipe._from_settings_called
|
|
||||||
|
|
||||||
def test_has_from_crawler_and_init(self):
|
def test_has_from_crawler_and_init(self):
|
||||||
class Pipeline(FilesPipeline):
|
class Pipeline(FilesPipeline):
|
||||||
_from_crawler_called = False
|
_from_crawler_called = False
|
||||||
|
|
|
||||||
|
|
@ -418,84 +418,6 @@ class TestBuildFromCrawler:
|
||||||
assert pipe._fingerprinter
|
assert pipe._fingerprinter
|
||||||
assert len(w) == 0
|
assert len(w) == 0
|
||||||
|
|
||||||
def test_has_old_init(self):
|
|
||||||
class Pipeline(UserDefinedPipeline):
|
|
||||||
def __init__(self):
|
|
||||||
super().__init__()
|
|
||||||
self._init_called = True
|
|
||||||
|
|
||||||
with warnings.catch_warnings(record=True) as w:
|
|
||||||
pipe = Pipeline.from_crawler(self.crawler)
|
|
||||||
assert pipe.crawler == self.crawler
|
|
||||||
assert pipe._fingerprinter
|
|
||||||
assert len(w) == 2
|
|
||||||
assert pipe._init_called
|
|
||||||
|
|
||||||
def test_has_from_settings(self):
|
|
||||||
class Pipeline(UserDefinedPipeline):
|
|
||||||
_from_settings_called = False
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_settings(cls, settings):
|
|
||||||
o = cls()
|
|
||||||
o._from_settings_called = True
|
|
||||||
return o
|
|
||||||
|
|
||||||
with warnings.catch_warnings(record=True) as w:
|
|
||||||
pipe = Pipeline.from_crawler(self.crawler)
|
|
||||||
assert pipe.crawler == self.crawler
|
|
||||||
assert pipe._fingerprinter
|
|
||||||
assert len(w) == 2
|
|
||||||
assert pipe._from_settings_called
|
|
||||||
|
|
||||||
def test_has_from_settings_and_from_crawler(self):
|
|
||||||
class Pipeline(UserDefinedPipeline):
|
|
||||||
_from_settings_called = False
|
|
||||||
_from_crawler_called = False
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_settings(cls, settings):
|
|
||||||
o = cls()
|
|
||||||
o._from_settings_called = True
|
|
||||||
return o
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_crawler(cls, crawler):
|
|
||||||
o = super().from_crawler(crawler)
|
|
||||||
o._from_crawler_called = True
|
|
||||||
return o
|
|
||||||
|
|
||||||
with warnings.catch_warnings(record=True) as w:
|
|
||||||
pipe = Pipeline.from_crawler(self.crawler)
|
|
||||||
assert pipe.crawler == self.crawler
|
|
||||||
assert pipe._fingerprinter
|
|
||||||
assert len(w) == 2
|
|
||||||
assert pipe._from_settings_called
|
|
||||||
assert pipe._from_crawler_called
|
|
||||||
|
|
||||||
def test_has_from_settings_and_init(self):
|
|
||||||
class Pipeline(UserDefinedPipeline):
|
|
||||||
_from_settings_called = False
|
|
||||||
|
|
||||||
def __init__(self, store_uri, settings):
|
|
||||||
super().__init__()
|
|
||||||
self._init_called = True
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_settings(cls, settings):
|
|
||||||
store_uri = settings["FILES_STORE"]
|
|
||||||
o = cls(store_uri, settings=settings)
|
|
||||||
o._from_settings_called = True
|
|
||||||
return o
|
|
||||||
|
|
||||||
with warnings.catch_warnings(record=True) as w:
|
|
||||||
pipe = Pipeline.from_crawler(self.crawler)
|
|
||||||
assert pipe.crawler == self.crawler
|
|
||||||
assert pipe._fingerprinter
|
|
||||||
assert len(w) == 2
|
|
||||||
assert pipe._from_settings_called
|
|
||||||
assert pipe._init_called
|
|
||||||
|
|
||||||
def test_has_from_crawler_and_init(self):
|
def test_has_from_crawler_and_init(self):
|
||||||
class Pipeline(UserDefinedPipeline):
|
class Pipeline(UserDefinedPipeline):
|
||||||
_from_crawler_called = False
|
_from_crawler_called = False
|
||||||
|
|
|
||||||
|
|
@ -1,20 +1,6 @@
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
|
|
||||||
def test_deprecated_twisted_version():
|
|
||||||
with warnings.catch_warnings(record=True) as warns:
|
|
||||||
from scrapy import ( # noqa: PLC0415 # pylint: disable=no-name-in-module
|
|
||||||
twisted_version,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert twisted_version is not None
|
|
||||||
assert isinstance(twisted_version, tuple)
|
|
||||||
assert (
|
|
||||||
"The scrapy.twisted_version attribute is deprecated, use twisted.version instead"
|
|
||||||
in warns[0].message.args
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_deprecated_concurrent_requests_per_ip_attribute():
|
def test_deprecated_concurrent_requests_per_ip_attribute():
|
||||||
with warnings.catch_warnings(record=True) as warns:
|
with warnings.catch_warnings(record=True) as warns:
|
||||||
from scrapy.settings.default_settings import ( # noqa: PLC0415
|
from scrapy.settings.default_settings import ( # noqa: PLC0415
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,6 @@ from scrapy.item import Field, Item
|
||||||
from scrapy.utils.misc import (
|
from scrapy.utils.misc import (
|
||||||
arg_to_iter,
|
arg_to_iter,
|
||||||
build_from_crawler,
|
build_from_crawler,
|
||||||
create_instance,
|
|
||||||
load_object,
|
load_object,
|
||||||
rel_has_nofollow,
|
rel_has_nofollow,
|
||||||
set_environ,
|
set_environ,
|
||||||
|
|
@ -97,98 +96,29 @@ class TestUtilsMisc:
|
||||||
assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
|
assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
|
||||||
assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]
|
assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]
|
||||||
|
|
||||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
|
||||||
def test_create_instance(self):
|
|
||||||
settings = mock.MagicMock()
|
|
||||||
crawler = mock.MagicMock(spec_set=["settings"])
|
|
||||||
args = (True, 100.0)
|
|
||||||
kwargs = {"key": "val"}
|
|
||||||
|
|
||||||
def _test_with_settings(mock, settings):
|
|
||||||
create_instance(mock, settings, None, *args, **kwargs)
|
|
||||||
if hasattr(mock, "from_crawler"):
|
|
||||||
assert mock.from_crawler.call_count == 0
|
|
||||||
if hasattr(mock, "from_settings"):
|
|
||||||
mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
|
|
||||||
assert mock.call_count == 0
|
|
||||||
else:
|
|
||||||
mock.assert_called_once_with(*args, **kwargs)
|
|
||||||
|
|
||||||
def _test_with_crawler(mock, settings, crawler):
|
|
||||||
create_instance(mock, settings, crawler, *args, **kwargs)
|
|
||||||
if hasattr(mock, "from_crawler"):
|
|
||||||
mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
|
|
||||||
if hasattr(mock, "from_settings"):
|
|
||||||
assert mock.from_settings.call_count == 0
|
|
||||||
assert mock.call_count == 0
|
|
||||||
elif hasattr(mock, "from_settings"):
|
|
||||||
mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
|
|
||||||
assert mock.call_count == 0
|
|
||||||
else:
|
|
||||||
mock.assert_called_once_with(*args, **kwargs)
|
|
||||||
|
|
||||||
# Check usage of correct constructor using four mocks:
|
|
||||||
# 1. with no alternative constructors
|
|
||||||
# 2. with from_settings() constructor
|
|
||||||
# 3. with from_crawler() constructor
|
|
||||||
# 4. with from_settings() and from_crawler() constructor
|
|
||||||
spec_sets = (
|
|
||||||
["__qualname__"],
|
|
||||||
["__qualname__", "from_settings"],
|
|
||||||
["__qualname__", "from_crawler"],
|
|
||||||
["__qualname__", "from_settings", "from_crawler"],
|
|
||||||
)
|
|
||||||
for specs in spec_sets:
|
|
||||||
m = mock.MagicMock(spec_set=specs)
|
|
||||||
_test_with_settings(m, settings)
|
|
||||||
m.reset_mock()
|
|
||||||
_test_with_crawler(m, settings, crawler)
|
|
||||||
|
|
||||||
# Check adoption of crawler settings
|
|
||||||
m = mock.MagicMock(spec_set=["__qualname__", "from_settings"])
|
|
||||||
create_instance(m, None, crawler, *args, **kwargs)
|
|
||||||
m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs)
|
|
||||||
|
|
||||||
with pytest.raises(
|
|
||||||
ValueError, match="Specify at least one of settings and crawler"
|
|
||||||
):
|
|
||||||
create_instance(m, None, None)
|
|
||||||
|
|
||||||
m.from_settings.return_value = None
|
|
||||||
with pytest.raises(TypeError):
|
|
||||||
create_instance(m, settings, None)
|
|
||||||
|
|
||||||
def test_build_from_crawler(self):
|
def test_build_from_crawler(self):
|
||||||
settings = mock.MagicMock()
|
|
||||||
crawler = mock.MagicMock(spec_set=["settings"])
|
crawler = mock.MagicMock(spec_set=["settings"])
|
||||||
args = (True, 100.0)
|
args = (True, 100.0)
|
||||||
kwargs = {"key": "val"}
|
kwargs = {"key": "val"}
|
||||||
|
|
||||||
def _test_with_crawler(mock, settings, crawler):
|
def _test_with_crawler(mock, crawler):
|
||||||
build_from_crawler(mock, crawler, *args, **kwargs)
|
build_from_crawler(mock, crawler, *args, **kwargs)
|
||||||
if hasattr(mock, "from_crawler"):
|
if hasattr(mock, "from_crawler"):
|
||||||
mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
|
mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
|
||||||
if hasattr(mock, "from_settings"):
|
|
||||||
assert mock.from_settings.call_count == 0
|
|
||||||
assert mock.call_count == 0
|
|
||||||
elif hasattr(mock, "from_settings"):
|
|
||||||
mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
|
|
||||||
assert mock.call_count == 0
|
assert mock.call_count == 0
|
||||||
else:
|
else:
|
||||||
mock.assert_called_once_with(*args, **kwargs)
|
mock.assert_called_once_with(*args, **kwargs)
|
||||||
|
|
||||||
# Check usage of correct constructor using three mocks:
|
# Check usage of correct constructor using 2 mocks:
|
||||||
# 1. with no alternative constructors
|
# 1. with no alternative constructors
|
||||||
# 2. with from_crawler() constructor
|
# 2. with from_crawler() constructor
|
||||||
# 3. with from_settings() and from_crawler() constructor
|
|
||||||
spec_sets = (
|
spec_sets = (
|
||||||
["__qualname__"],
|
["__qualname__"],
|
||||||
["__qualname__", "from_crawler"],
|
["__qualname__", "from_crawler"],
|
||||||
["__qualname__", "from_settings", "from_crawler"],
|
|
||||||
)
|
)
|
||||||
for specs in spec_sets:
|
for specs in spec_sets:
|
||||||
m = mock.MagicMock(spec_set=specs)
|
m = mock.MagicMock(spec_set=specs)
|
||||||
_test_with_crawler(m, settings, crawler)
|
_test_with_crawler(m, crawler)
|
||||||
m.reset_mock()
|
m.reset_mock()
|
||||||
|
|
||||||
# Check adoption of crawler
|
# Check adoption of crawler
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,6 @@ from scrapy.utils.python import (
|
||||||
MutableAsyncChain,
|
MutableAsyncChain,
|
||||||
MutableChain,
|
MutableChain,
|
||||||
binary_is_text,
|
binary_is_text,
|
||||||
equal_attributes,
|
|
||||||
get_func_args,
|
get_func_args,
|
||||||
memoizemethod_noargs,
|
memoizemethod_noargs,
|
||||||
to_bytes,
|
to_bytes,
|
||||||
|
|
@ -150,56 +149,6 @@ def test_binaryistext(value: bytes, expected: bool) -> None:
|
||||||
assert binary_is_text(value) is expected
|
assert binary_is_text(value) is expected
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
|
||||||
def test_equal_attributes():
|
|
||||||
class Obj:
|
|
||||||
pass
|
|
||||||
|
|
||||||
a = Obj()
|
|
||||||
b = Obj()
|
|
||||||
# no attributes given return False
|
|
||||||
assert not equal_attributes(a, b, [])
|
|
||||||
# nonexistent attributes
|
|
||||||
assert not equal_attributes(a, b, ["x", "y"])
|
|
||||||
|
|
||||||
a.x = 1
|
|
||||||
b.x = 1
|
|
||||||
# equal attribute
|
|
||||||
assert equal_attributes(a, b, ["x"])
|
|
||||||
|
|
||||||
b.y = 2
|
|
||||||
# obj1 has no attribute y
|
|
||||||
assert not equal_attributes(a, b, ["x", "y"])
|
|
||||||
|
|
||||||
a.y = 2
|
|
||||||
# equal attributes
|
|
||||||
assert equal_attributes(a, b, ["x", "y"])
|
|
||||||
|
|
||||||
a.y = 1
|
|
||||||
# different attributes
|
|
||||||
assert not equal_attributes(a, b, ["x", "y"])
|
|
||||||
|
|
||||||
# test callable
|
|
||||||
a.meta = {}
|
|
||||||
b.meta = {}
|
|
||||||
assert equal_attributes(a, b, ["meta"])
|
|
||||||
|
|
||||||
# compare ['meta']['a']
|
|
||||||
a.meta["z"] = 1
|
|
||||||
b.meta["z"] = 1
|
|
||||||
|
|
||||||
get_z = operator.itemgetter("z")
|
|
||||||
get_meta = operator.attrgetter("meta")
|
|
||||||
|
|
||||||
def compare_z(obj):
|
|
||||||
return get_z(get_meta(obj))
|
|
||||||
|
|
||||||
assert equal_attributes(a, b, [compare_z, "x"])
|
|
||||||
# fail z equality
|
|
||||||
a.meta["z"] = 2
|
|
||||||
assert not equal_attributes(a, b, [compare_z, "x"])
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_func_args():
|
def test_get_func_args():
|
||||||
def f1(a, b, c):
|
def f1(a, b, c):
|
||||||
pass
|
pass
|
||||||
|
|
|
||||||
|
|
@ -1,32 +1,22 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import warnings
|
|
||||||
from hashlib import sha1
|
from hashlib import sha1
|
||||||
from weakref import WeakKeyDictionary
|
from weakref import WeakKeyDictionary
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
|
||||||
from scrapy.http import Request
|
from scrapy.http import Request
|
||||||
from scrapy.utils.python import to_bytes
|
from scrapy.utils.python import to_bytes
|
||||||
from scrapy.utils.request import (
|
from scrapy.utils.request import (
|
||||||
_fingerprint_cache,
|
_fingerprint_cache,
|
||||||
fingerprint,
|
fingerprint,
|
||||||
request_authenticate,
|
|
||||||
request_httprepr,
|
request_httprepr,
|
||||||
request_to_curl,
|
request_to_curl,
|
||||||
)
|
)
|
||||||
from scrapy.utils.test import get_crawler
|
from scrapy.utils.test import get_crawler
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
|
||||||
def test_request_authenticate():
|
|
||||||
r = Request("http://www.example.com")
|
|
||||||
request_authenticate(r, "someuser", "somepass")
|
|
||||||
assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M="
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("r", "expected"),
|
("r", "expected"),
|
||||||
[
|
[
|
||||||
|
|
@ -237,25 +227,13 @@ class TestFingerprint:
|
||||||
|
|
||||||
|
|
||||||
class TestRequestFingerprinter:
|
class TestRequestFingerprinter:
|
||||||
def test_default_implementation(self):
|
def test_fingerprint(self):
|
||||||
crawler = get_crawler()
|
crawler = get_crawler()
|
||||||
request = Request("https://example.com")
|
request = Request("https://example.com")
|
||||||
assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
|
assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
|
||||||
request
|
request
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_deprecated_implementation(self):
|
|
||||||
settings = {
|
|
||||||
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
|
|
||||||
}
|
|
||||||
with warnings.catch_warnings(record=True) as logged_warnings:
|
|
||||||
crawler = get_crawler(settings_dict=settings)
|
|
||||||
request = Request("https://example.com")
|
|
||||||
assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
|
|
||||||
request
|
|
||||||
)
|
|
||||||
assert logged_warnings
|
|
||||||
|
|
||||||
|
|
||||||
class TestCustomRequestFingerprinter:
|
class TestCustomRequestFingerprinter:
|
||||||
def test_include_headers(self):
|
def test_include_headers(self):
|
||||||
|
|
@ -343,57 +321,6 @@ class TestCustomRequestFingerprinter:
|
||||||
fingerprint = crawler.request_fingerprinter.fingerprint(request)
|
fingerprint = crawler.request_fingerprinter.fingerprint(request)
|
||||||
assert fingerprint == settings["FINGERPRINT"]
|
assert fingerprint == settings["FINGERPRINT"]
|
||||||
|
|
||||||
def test_from_settings(self):
|
|
||||||
class RequestFingerprinter:
|
|
||||||
@classmethod
|
|
||||||
def from_settings(cls, settings):
|
|
||||||
return cls(settings)
|
|
||||||
|
|
||||||
def __init__(self, settings):
|
|
||||||
self._fingerprint = settings["FINGERPRINT"]
|
|
||||||
|
|
||||||
def fingerprint(self, request):
|
|
||||||
return self._fingerprint
|
|
||||||
|
|
||||||
settings = {
|
|
||||||
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
|
|
||||||
"FINGERPRINT": b"fingerprint",
|
|
||||||
}
|
|
||||||
with warnings.catch_warnings():
|
|
||||||
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
|
|
||||||
crawler = get_crawler(settings_dict=settings)
|
|
||||||
|
|
||||||
request = Request("http://www.example.com")
|
|
||||||
fingerprint = crawler.request_fingerprinter.fingerprint(request)
|
|
||||||
assert fingerprint == settings["FINGERPRINT"]
|
|
||||||
|
|
||||||
def test_from_crawler_and_settings(self):
|
|
||||||
class RequestFingerprinter:
|
|
||||||
# This method is ignored due to the presence of from_crawler
|
|
||||||
@classmethod
|
|
||||||
def from_settings(cls, settings):
|
|
||||||
return cls(settings)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_crawler(cls, crawler):
|
|
||||||
return cls(crawler)
|
|
||||||
|
|
||||||
def __init__(self, crawler):
|
|
||||||
self._fingerprint = crawler.settings["FINGERPRINT"]
|
|
||||||
|
|
||||||
def fingerprint(self, request):
|
|
||||||
return self._fingerprint
|
|
||||||
|
|
||||||
settings = {
|
|
||||||
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
|
|
||||||
"FINGERPRINT": b"fingerprint",
|
|
||||||
}
|
|
||||||
crawler = get_crawler(settings_dict=settings)
|
|
||||||
|
|
||||||
request = Request("http://www.example.com")
|
|
||||||
fingerprint = crawler.request_fingerprinter.fingerprint(request)
|
|
||||||
assert fingerprint == settings["FINGERPRINT"]
|
|
||||||
|
|
||||||
|
|
||||||
class TestRequestToCurl:
|
class TestRequestToCurl:
|
||||||
def _test_request(self, request_object, expected_curl_command):
|
def _test_request(self, request_object, expected_curl_command):
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue