RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into checksum-algorithm

This commit is contained in:
Adrian Chaves 2026-08-10 23:45:14 +02:00
parent cee0ce3344 cf6ef74c79
commit f56eac8607
87 changed files with 2772 additions and 2390 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.pre-commit-config.yaml
View File

@ -27,7 +27,7 @@ repos:
hooks:
- id: sphinx-lint
- repo: https://github.com/scrapy/sphinx-scrapy
rev: 0.8.10
rev: 0.8.11
hooks:
- id: sphinx-scrapy
- repo: https://github.com/zizmorcore/zizmor-pre-commit

9
docs/_ext/scrapydocs.py
View File

@ -137,14 +137,6 @@ def source_role(
return [node], []
def issue_role(
name, rawtext, text: str, lineno, inliner, options=None, content=None
) -> tuple[list[Any], list[Any]]:
ref = "https://github.com/scrapy/scrapy/issues/" + text
node = nodes.reference(rawtext, "issue " + text, refuri=ref)
return [node], []
def commit_role(
name, rawtext, text: str, lineno, inliner, options=None, content=None
) -> tuple[list[Any], list[Any]]:
@ -164,7 +156,6 @@ def rev_role(
def setup(app: Sphinx) -> dict[str, Any]:
app.add_role("source", source_role)
app.add_role("commit", commit_role)
app.add_role("issue", issue_role)
app.add_role("rev", rev_role)
app.add_node(

4
docs/faq.rst
View File

@ -141,7 +141,7 @@ middleware with a :ref:`custom downloader middleware
- If you can meet the installation requirements, use pyre2_ instead of
Python’s re_ to compile your URL-filtering regular expression. See
:issue:`1908`.
:gh:`1908`.
See also `other suggestions at StackOverflow
<https://stackoverflow.com/q/36440681>`__.
@ -419,7 +419,7 @@ Running ``runspider`` I get ``error: No spider found in file: <filename>``
This may happen if your Scrapy project has a spider module with a name that
conflicts with the name of one of the `Python standard library modules`_, such
as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed.
See :issue:`2680`.
See :gh:`2680`.
.. _has been reported: https://github.com/scrapy/scrapy/issues/2905

4002
docs/news.rst
View File

File diff suppressed because it is too large Load Diff

2
docs/requirements.in
View File

@ -6,4 +6,4 @@ sphinx-notfound-page
sphinx-reredirects
sphinx-rtd-theme
sphinx-rtd-dark-mode
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.11

2
docs/requirements.txt
View File

@ -156,7 +156,7 @@ sphinx-rtd-theme==3.1.0
# via
# -r docs/requirements.in
# sphinx-rtd-dark-mode
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@6f8e5e0bbd171a857da480f7188f2a205041cb60
# via -r docs/requirements.in
sphinx-sitemap==2.9.0
# via sphinx-scrapy

35
docs/topics/download-handlers.rst
View File

@ -130,17 +130,23 @@ using different handlers.
Here is a comparison of some features of the built-in HTTP handlers, see the
individual handler docs for more differences:
================== ================= ===================== ====================
Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler
================== ================= ===================== ====================
Requires asyncio No No Yes
Requires a reactor Yes Yes No
HTTP/1.1 No Yes Yes
HTTP/2 Yes No Yes
TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl``
HTTP proxies No Yes Yes
SOCKS proxies No No Yes
================== ================= ===================== ====================
=================== ================= ===================== ====================
Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler
=================== ================= ===================== ====================
Requires asyncio No No Yes
Requires a reactor Yes Yes No
HTTP/1.1 No Yes Yes
HTTP/2 Yes No Yes
TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl``
HTTP proxies No Yes Yes
SOCKS proxies No No Yes
Bad header handling Not applicable Skip bad Fail
=================== ================= ===================== ====================
Bad header handling is what a handler does when a response has a bad header
line, e.g. one with no colon in it, which some servers send. Handlers that skip
bad header lines, like web browsers do, still parse the header lines that follow
them; other handlers also lose those, or cannot download such responses at all.
You can find additional HTTP download handlers in the
scrapy-download-handlers-incubator_ package. This package is made by the Scrapy
@ -191,6 +197,7 @@ Features and limitations
HTTP proxies No (not implemented)
SOCKS proxies No (not supported by the library)
HTTP/2 Yes
Bad header handling Not applicable (HTTP/2 only)
``response.certificate`` :class:`twisted.internet.ssl.Certificate` object
Per-request ``bindaddress`` Yes
TLS implementation ``pyOpenSSL``/``cryptography``
@ -239,11 +246,16 @@ Features and limitations
HTTP proxies Yes
SOCKS proxies No (not supported by the library)
HTTP/2 No (implemented as a separate handler)
Bad header handling Skip bad, like web browsers do
``response.certificate`` :class:`twisted.internet.ssl.Certificate` object
Per-request ``bindaddress`` Yes
TLS implementation ``pyOpenSSL``/``cryptography``
=========================== ================================================
.. versionchanged:: VERSION
Bad header lines with no colon in them are now skipped, instead of making
the whole response impossible to download.
Other limitations:
- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`
@ -297,6 +309,7 @@ Features and limitations
HTTP proxies Yes
SOCKS proxies Yes (SOCKS5)
HTTP/2 Yes
Bad header handling Fail (not supported by the library)
``response.certificate`` DER bytes
Per-request ``bindaddress`` No (not supported by the library)
TLS implementation Standard library ``ssl``

4
docs/topics/extensions.rst
View File

@ -374,8 +374,8 @@ This extension periodically logs rich stat data as a JSON object::
"elapsed": 360.008903,
"log_interval": 60.0,
"log_interval_real": 60.006694,
"start_time": "2023-08-03 23:24:57",
"utcnow": "2023-08-03 23:30:57"
"start_time": "2023-08-03T23:24:57.148903+00:00",
"utcnow": "2023-08-03T23:30:57.157806+00:00"
}
}

32
docs/topics/feed-exports.rst
View File

@ -104,7 +104,8 @@ storage backend types which are defined by the URI scheme.
The storages backends supported out of the box are:
- :ref:`topics-feed-storage-fs`
- :ref:`topics-feed-storage-ftp`
- :ref:`feed-storage-ftp`
- :ref:`feed-storage-ftps`
- :ref:`topics-feed-storage-s3` (requires the :ref:`s3 <extras>` extra)
- :ref:`topics-feed-storage-gcs` (requires the :ref:`gcs <extras>` extra)
- :ref:`topics-feed-storage-stdout`
@ -168,6 +169,7 @@ you specify a path (e.g. ``/tmp/export.csv``).
Alternatively you can also use a :class:`pathlib.Path` object.
.. _topics-feed-storage-ftp:
.. _feed-storage-ftp:
FTP
---
@ -178,6 +180,9 @@ The feeds are stored in a FTP server.
- Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
- Required external libraries: none
FTP sends credentials and data in cleartext. Use :ref:`feed-storage-ftps`
instead where possible.
FTP supports two different connection modes: `active or passive
<https://stackoverflow.com/a/1699163>`_. Scrapy uses the passive connection
mode by default. To use the active connection mode instead, set the
@ -192,6 +197,28 @@ storage backend is: ``True``.
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _feed-storage-ftps:
FTPS
----
The feeds are stored in a FTP server, over a TLS connection, with the
certificate of the server verified.
.. versionadded:: VERSION
- URI scheme: ``ftps``
- Example URI: ``ftps://user:pass@ftp.example.com/path/to/export.csv``
- Required external libraries: none
See :ref:`feed-storage-ftp` for connection modes, the ``overwrite`` default and
file delivery.
.. note:: For SFTP, an unrelated protocol built on SSH, use
`scrapy-feedexporter-sftp
<https://github.com/scrapy-plugins/scrapy-feedexporter-sftp>`_.
.. _topics-feed-storage-s3:
S3
@ -502,7 +529,7 @@ as a fallback value if that key is not provided for a specific feed definition:
- :ref:`topics-feed-storage-fs`: ``False``
- :ref:`topics-feed-storage-ftp`: ``True``
- :ref:`feed-storage-ftp` and :ref:`feed-storage-ftps`: ``True``
.. note:: Some FTP servers may not support appending to files (the
``APPE`` FTP command).
@ -624,6 +651,7 @@ Default:
"s3": "scrapy.extensions.feedexport.S3FeedStorage",
"gs": "scrapy.extensions.feedexport.GCSFeedStorage",
"ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
"ftps": "scrapy.extensions.feedexport.FTPFeedStorage",
}
A dict containing the built-in feed storage backends supported by Scrapy. You

2
docs/topics/settings.rst
View File

@ -1393,7 +1393,7 @@ FEED_TEMPDIR
Default: ``None``
The Feed Temp dir allows you to set a custom folder to save crawler
temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
temporary files before uploading with :ref:`FTP feed storage <feed-storage-ftp>` and
:ref:`Amazon S3 <topics-feed-storage-s3>`.
.. setting:: FEED_STORAGE_GCS_ACL

5
scrapy/core/downloader/__init__.py
View File

@ -28,6 +28,7 @@ from scrapy.utils.defer import (
maybe_deferred_to_future,
)
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
from collections.abc import Generator
@ -99,8 +100,8 @@ class Downloader:
# AUTOTHROTTLE_START_DELAY.
self._delay: float = self.settings.getfloat("DOWNLOAD_DELAY")
self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")
self.middleware: DownloaderMiddlewareManager = (
DownloaderMiddlewareManager.from_crawler(crawler)
self.middleware: DownloaderMiddlewareManager = build_from_crawler(
DownloaderMiddlewareManager, crawler
)
self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = None
self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict(

86
scrapy/core/downloader/handlers/http11.py
View File

@ -17,12 +17,19 @@ from twisted.internet.defer import Deferred, succeed
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.protocol import Factory, Protocol, connectionDone
from twisted.python.failure import Failure
from twisted.web._newclient import (
HEADER,
STATUS,
HTTP11ClientProtocol,
HTTPClientParser,
)
from twisted.web.client import (
URI,
Agent,
HTTPConnectionPool,
ResponseDone,
ResponseFailed,
_HTTP11ClientFactory,
)
from twisted.web.client import Response as TxResponse
from twisted.web.http import PotentialDataLoss, _DataLoss
@ -60,7 +67,8 @@ from ._base_http import BaseHttpDownloadHandler
if TYPE_CHECKING:
from twisted.internet.base import ReactorBase
from twisted.internet.interfaces import IConsumer
from twisted.internet.interfaces import IAddress, IConsumer
from twisted.web._newclient import Request as TxRequest
# typing.NotRequired requires Python 3.11
from typing_extensions import NotRequired
@ -95,7 +103,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
self._pool.maxPersistentPerHost = crawler.settings.getint(
"CONCURRENT_REQUESTS_PER_DOMAIN"
)
self._pool._factory.noisy = False
self._pool._factory = _LenientHTTP11ClientFactory
self._contextFactory: IPolicyForHTTPS = _load_context_factory_from_settings(
crawler
@ -740,3 +748,77 @@ class _ResponseReader(Protocol):
reason = Failure(exc)
self._finished.errback(reason)
class _LenientHTTPClientParser(HTTPClientParser):
"""Response parser that skips bad response header lines, those with no
colon in them, instead of failing to parse the whole response.
Some servers send such lines, and web browsers skip them and keep parsing
the header lines that follow. See
https://github.com/scrapy/scrapy/issues/210.
"""
def lineReceived(self, line: bytes) -> None:
# A copy of twisted.web._newclient.HTTPParser.lineReceived() where the
# header name and value are only extracted from header lines that have
# a colon.
# Handle the normal CR LF case.
if line[-1:] == b"\r":
line = line[:-1]
if self.state == STATUS:
self.statusReceived(line) # type: ignore[no-untyped-call]
self.state = HEADER
return
# HEADER is the only other state in which lines are received, as the
# parser switches to raw mode for the response body.
if not line or line[0] not in b" \t":
if self._partialHeader is not None:
header = b"".join(self._partialHeader)
if b":" in header:
name, value = header.split(b":", 1)
self.headerReceived(name, value.strip()) # type: ignore[no-untyped-call]
else:
logger.debug(
f"Skipping the bad response header line {header!r}, as "
f"it has no colon."
)
if not line:
# Empty line means the header section is over.
self.allHeadersReceived() # type: ignore[no-untyped-call]
else:
# Line not beginning with LWS is another header.
self._partialHeader = [line]
else:
# A line beginning with LWS is a continuation of a header begun on
# a previous line.
self._partialHeader.append(line) # type: ignore[union-attr]
class _LenientHTTP11ClientProtocol(HTTP11ClientProtocol):
"""Protocol that parses responses with :class:`_LenientHTTPClientParser`."""
def request(self, request: TxRequest) -> Deferred[IResponse]:
d: Deferred[IResponse] = super().request(request)
# HTTP11ClientProtocol.request() hardcodes the parser class, so the
# only way to use a different one is to replace the class of the parser
# object that it creates. This is safe because
# _LenientHTTPClientParser defines no additional state. The parser is
# always there because HTTPConnectionPool only reuses connections whose
# protocol is in the QUIESCENT state, for which request() always
# creates a parser.
assert self._parser is not None
self._parser.__class__ = _LenientHTTPClientParser
return d
class _LenientHTTP11ClientFactory(_HTTP11ClientFactory):
"""Factory that builds :class:`_LenientHTTP11ClientProtocol` protocols."""
noisy = False
def buildProtocol(self, addr: IAddress | None) -> HTTP11ClientProtocol:
return _LenientHTTP11ClientProtocol(self._quiescentCallback) # type: ignore[no-untyped-call]

12
scrapy/core/scraper.py
View File

@ -36,7 +36,11 @@ from scrapy.utils.defer import (
)
from scrapy.utils.deprecate import method_is_overridden
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.misc import (
build_from_crawler,
load_object,
warn_on_generator_with_return_value,
)
from scrapy.utils.python import global_object_name
from scrapy.utils.spider import iterate_spider_output
@ -102,13 +106,13 @@ class Slot:
class Scraper:
def __init__(self, crawler: Crawler) -> None:
self.slot: Slot | None = None
self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler(
crawler
self.spidermw: SpiderMiddlewareManager = build_from_crawler(
SpiderMiddlewareManager, crawler
)
itemproc_cls: type[ItemPipelineManager] = load_object(
crawler.settings["ITEM_PROCESSOR"]
)
self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler)
self.itemproc: ItemPipelineManager = build_from_crawler(itemproc_cls, crawler)
self._itemproc_has_async: dict[str, bool] = {}
for method in [
"open_spider",

4
scrapy/crawler.py
View File

@ -156,7 +156,7 @@ class Crawler:
self.stats = load_object(self.settings["STATS_CLASS"])(self)
lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
self.logformatter = lf_cls.from_crawler(self)
self.logformatter = build_from_crawler(lf_cls, self)
self.request_fingerprinter = build_from_crawler(
load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
@ -200,7 +200,7 @@ class Crawler:
logger.debug("Not using a Twisted reactor")
self._apply_reactorless_default_settings()
self.extensions = ExtensionManager.from_crawler(self)
self.extensions = build_from_crawler(ExtensionManager, self)
self.settings.freeze()
d = dict(overridden_settings(self.settings))

6
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -18,7 +18,7 @@ from scrapy.http.request import NO_CALLBACK
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
# typing.Self requires Python 3.11
@ -49,7 +49,7 @@ class RobotsTxtMiddleware:
)
# check if parser dependencies are met, this should throw an error otherwise.
self._parserimpl.from_crawler(self.crawler, b"")
build_from_crawler(self._parserimpl, self.crawler, b"")
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
@ -120,7 +120,7 @@ class RobotsTxtMiddleware:
) -> None:
self._stats.inc_value("robotstxt/response_count")
self._stats.inc_value(f"robotstxt/response_status_count/{response.status}")
rp = self._parserimpl.from_crawler(self.crawler, response.body)
rp = build_from_crawler(self._parserimpl, self.crawler, response.body)
await self.crawler.signals.send_catch_log_async(
signal=signals.robots_parsed,
robotparser=rp,

2
scrapy/extensions/feedexport.py
View File

@ -363,6 +363,7 @@ class FTPFeedStorage(BlockingFeedStorage):
self.username: str = u.username or ""
self.password: str = unquote(u.password or "")
self.path: str = u.path
self.tls: bool = u.scheme == "ftps"
self.use_active_mode: bool = use_active_mode
self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)
@ -390,6 +391,7 @@ class FTPFeedStorage(BlockingFeedStorage):
password=self.password,
use_active_mode=self.use_active_mode,
overwrite=self.overwrite,
tls=self.tls,
)

3
scrapy/extensions/memusage.py
View File

@ -19,6 +19,7 @@ from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
from scrapy.utils.defer import _schedule_coro
from scrapy.utils.engine import get_engine_status
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
from twisted.internet.task import LoopingCall
@ -57,7 +58,7 @@ class MemoryUsage:
category=ScrapyDeprecationWarning,
stacklevel=2,
)
self.mail = MailSender.from_crawler(crawler)
self.mail = build_from_crawler(MailSender, crawler)
self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024
self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024

3
scrapy/extensions/statsmailer.py
View File

@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.mail import MailSender
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
@ -41,7 +42,7 @@ class StatsMailer:
recipients: list[str] = crawler.settings.getlist("STATSMAILER_RCPTS")
if not recipients:
raise NotConfigured
mail: MailSender = MailSender.from_crawler(crawler)
mail: MailSender = build_from_crawler(MailSender, crawler)
o = cls(crawler.stats, recipients, mail)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o

26
scrapy/pipelines/files.py
View File

@ -27,9 +27,7 @@ from twisted.internet.defer import Deferred, maybeDeferred
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.media import (
FileException as FileException, # noqa: PLC0414 # re-exported for backward compatibility
)
from scrapy.pipelines.media import FileException as _FileException
from scrapy.pipelines.media import (
FileInfo,
FileInfoOrError,
@ -668,7 +666,7 @@ class FilesPipeline(MediaPipeline):
f"{request} referred in <{referer}>: {failure.value}",
extra={"spider": info.spider},
)
raise FileException
raise _FileException
async def media_downloaded(
self,
@ -687,7 +685,7 @@ class FilesPipeline(MediaPipeline):
{"status": response.status, "request": request, "referer": referer},
extra={"spider": info.spider},
)
raise FileException("download-error")
raise _FileException("download-error")
if not response.body:
logger.warning(
@ -696,7 +694,7 @@ class FilesPipeline(MediaPipeline):
{"request": request, "referer": referer},
extra={"spider": info.spider},
)
raise FileException("empty-content")
raise _FileException("empty-content")
status = "cached" if "cached" in response.flags else "downloaded"
logger.debug(
@ -712,7 +710,7 @@ class FilesPipeline(MediaPipeline):
checksum: str = await ensure_awaitable(
self.file_downloaded(response, request, info, item=item)
)
except FileException as exc:
except _FileException as exc:
logger.warning(
"File (error): Error processing file from %(request)s "
"referred in <%(referer)s>: %(errormsg)s",
@ -729,7 +727,7 @@ class FilesPipeline(MediaPipeline):
exc_info=True,
extra={"spider": info.spider},
)
raise FileException(str(exc)) from exc
raise _FileException(str(exc)) from exc
return {
"url": request.url,
@ -812,3 +810,15 @@ class FilesPipeline(MediaPipeline):
if media_type:
media_ext = cast("str", mimetypes.guess_extension(media_type))
return f"full/{media_guid}{media_ext}"
def __getattr__(name: str) -> Any:
if name == "FileException":
warnings.warn(
"scrapy.pipelines.files.FileException is deprecated, use "
"scrapy.pipelines.media.FileException instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return _FileException
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

2
scrapy/pipelines/images.py
View File

@ -19,13 +19,13 @@ from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import (
FileException,
FilesPipeline,
GCSFilesStore,
S3FilesStore,
_checksum,
_checksum_algorithm,
)
from scrapy.pipelines.media import FileException
from scrapy.utils.defer import ensure_awaitable
from scrapy.utils.python import to_bytes

1
scrapy/settings/default_settings.py
View File

@ -378,6 +378,7 @@ FEED_STORAGES_BASE = {
"": "scrapy.extensions.feedexport.FileFeedStorage",
"file": "scrapy.extensions.feedexport.FileFeedStorage",
"ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
"ftps": "scrapy.extensions.feedexport.FTPFeedStorage",
"gs": "scrapy.extensions.feedexport.GCSFeedStorage",
"s3": "scrapy.extensions.feedexport.S3FeedStorage",
"stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",

16
scrapy/utils/ftp.py
View File

@ -1,7 +1,8 @@
import posixpath
from contextlib import closing
from ftplib import FTP, error_perm
from ftplib import FTP, FTP_TLS, error_perm
from posixpath import dirname
from ssl import create_default_context
from typing import IO
@ -29,13 +30,20 @@ def ftp_store_file(
password: str,
use_active_mode: bool = False,
overwrite: bool = True,
tls: bool = False,
) -> None:
"""Opens a FTP connection with passed credentials,sets current directory
to the directory extracted from given path, then uploads the file to server
"""Opens a FTP connection with passed credentials, sets current directory
to the directory extracted from given path, then uploads the file to server.
If *tls* is ``True``, the connection is secured with TLS (FTPS), and the
certificate of the server is verified.
"""
with FTP() as ftp, closing(file):
ftp = FTP_TLS(context=create_default_context()) if tls else FTP()
with ftp, closing(file):
ftp.connect(host, port)
ftp.login(username, password)
if isinstance(ftp, FTP_TLS):
ftp.prot_p()
if use_active_mode:
ftp.set_pasv(False)
file.seek(0)

20
scrapy/utils/log.py
View File

@ -2,7 +2,9 @@ from __future__ import annotations
import logging
import pprint
import re
import sys
import warnings
from collections.abc import MutableMapping
from logging.config import dictConfig
from typing import TYPE_CHECKING, Any, cast
@ -12,6 +14,7 @@ from twisted.python import log as twisted_log
from twisted.python.failure import Failure
import scrapy
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
from scrapy.utils.versions import get_versions
@ -242,6 +245,9 @@ class LogCounterHandler(logging.Handler):
self.crawler.stats.inc_value(sname)
_MSG_MAPPING_PLACEHOLDER = re.compile(r"%\(\w+\)")
def logformatter_adapter(
logkws: LogFormatterResult,
) -> tuple[Any, ...]:
@ -257,6 +263,20 @@ def logformatter_adapter(
# argument, so empty args are left out. Tuple args become one positional
# argument each, while a dict is a single positional argument.
if not args:
if _MSG_MAPPING_PLACEHOLDER.search(message):
# The log formatter method has already returned, so there is no
# frame of it left in the stack to point at. msg is part of the
# warning message instead, so that each offending method gets its
# own warning.
warnings.warn(
f"A log formatter method returned msg {message!r} with "
f"%(name)s placeholders and no args. Interpolating msg with "
f"the returned dict is deprecated, return those values under "
f"args instead.",
ScrapyDeprecationWarning,
stacklevel=1,
)
return (level, message, logkws)
return (level, message)
if isinstance(args, tuple):
return (level, message, *args)

11
scrapy/utils/serialize.py
View File

@ -10,18 +10,11 @@ from scrapy.http import Request, Response
class ScrapyJSONEncoder(json.JSONEncoder):
DATE_FORMAT = "%Y-%m-%d"
TIME_FORMAT = "%H:%M:%S"
def default(self, o: Any) -> Any:
if isinstance(o, set):
return list(o)
if isinstance(o, datetime.datetime):
return o.strftime(f"{self.DATE_FORMAT} {self.TIME_FORMAT}")
if isinstance(o, datetime.date):
return o.strftime(self.DATE_FORMAT)
if isinstance(o, datetime.time):
return o.strftime(self.TIME_FORMAT)
if isinstance(o, (datetime.datetime, datetime.date, datetime.time)):
return o.isoformat()
if isinstance(o, decimal.Decimal):
return str(o)
if isinstance(o, defer.Deferred):

6
tests/keys/__init__.py
View File

@ -1,4 +1,5 @@
from datetime import datetime, timedelta, timezone
from ipaddress import IPv4Address
from pathlib import Path
from cryptography.hazmat.backends import default_backend
@ -12,6 +13,7 @@ from cryptography.hazmat.primitives.serialization import (
from cryptography.x509 import (
CertificateBuilder,
DNSName,
IPAddress,
Name,
NameAttribute,
SubjectAlternativeName,
@ -53,7 +55,9 @@ def generate_keys():
.not_valid_before(datetime.now(tz=timezone.utc))
.not_valid_after(datetime.now(tz=timezone.utc) + timedelta(days=10))
.add_extension(
SubjectAlternativeName([DNSName("localhost")]),
SubjectAlternativeName(
[DNSName("localhost"), IPAddress(IPv4Address("127.0.0.1"))]
),
critical=False,
)
.sign(key, SHA256(), default_backend())

32
tests/mockserver/ftp.py
View File

@ -10,7 +10,7 @@ from tempfile import mkdtemp
from typing import TYPE_CHECKING
from pyftpdlib.authorizers import DummyAuthorizer
from pyftpdlib.handlers import FTPHandler
from pyftpdlib.handlers import FTPHandler, TLS_FTPHandler
from pyftpdlib.servers import FTPServer
from tests.utils import get_script_run_env
@ -25,28 +25,32 @@ if TYPE_CHECKING:
class MockFTPServer:
"""Creates an FTP server on a random port with a default passwordless user
(anonymous) and a temporary root path that you can read from the
:attr:`path` attribute."""
:attr:`path` attribute.
If *tls* is ``True``, the server requires FTPS, using the test certificate
from :file:`tests/keys`.
"""
proc: Popen[str]
port: int
path: Path
def __init__(self) -> None:
def __init__(self, tls: bool = False) -> None:
self.host: str = "127.0.0.1"
self.tls: bool = tls
def __enter__(self) -> Self:
self.path = Path(mkdtemp())
self.proc = Popen(
[sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)],
[sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)]
+ (["--tls"] if self.tls else []),
stderr=PIPE,
env=get_script_run_env(),
text=True,
)
assert self.proc.stderr is not None
for line in self.proc.stderr:
if "starting FTP server" in line and (
m := re.search(r"starting FTP server on ([^ :]+):(\d+),", line)
):
if m := re.search(r"starting FTPS? .*on ([^ :]+):(\d+),", line):
self.port = int(m.group(2))
break
else:
@ -68,18 +72,28 @@ class MockFTPServer:
self.proc.communicate()
def url(self, path: str) -> str:
return f"ftp://{self.host}:{self.port}/{path}"
scheme = "ftps" if self.tls else "ftp"
return f"{scheme}://{self.host}:{self.port}/{path}"
def main() -> None:
parser = ArgumentParser()
parser.add_argument("-d", "--directory", required=True)
parser.add_argument("--tls", action="store_true")
args = parser.parse_args()
authorizer = DummyAuthorizer()
full_permissions = "elradfmwMT"
authorizer.add_anonymous(args.directory, perm=full_permissions)
handler = FTPHandler
if args.tls:
keys = Path(__file__).parent.parent / "keys"
handler = TLS_FTPHandler
handler.certfile = str(keys / "localhost.crt")
handler.keyfile = str(keys / "localhost.key")
handler.tls_control_required = True
handler.tls_data_required = True
else:
handler = FTPHandler
handler.authorizer = authorizer
address = ("127.0.0.1", 0)
server = FTPServer(address, handler)

2
tests/mockserver/http.py
View File

@ -11,6 +11,7 @@ from tests import tests_datadir
from .http_base import BaseMockServer, main_factory
from .http_resources import (
ArbitraryLengthPayloadResource,
BadHeader,
BaseResource,
BrokenChunkedResource,
BrokenDownloadResource,
@ -52,6 +53,7 @@ class Root(BaseResource):
put_child(self, b"partial", Partial())
put_child(self, b"drop", Drop())
put_child(self, b"raw", Raw())
put_child(self, b"bad-header", BadHeader())
put_child(self, b"echo", Echo())
put_child(self, b"payload", PayloadResource())
put_child(self, b"alpayload", ArbitraryLengthPayloadResource())

33
tests/mockserver/http_resources.py
View File

@ -210,6 +210,39 @@ class Raw(LeafResource):
request.finish()
class BadHeader(LeafResource):
"""Sends a response with a bad header line, one with no colon in it, like
some servers do, between two good ones.
One of the good header lines is split into two lines, so that handling of
such headers is also covered.
"""
response = (
b"HTTP/1.1 200 OK\r\n"
b"Content-Length: 5\r\n"
b"Content-Type: text/html\r\n"
b"X-Folded-Header: one\r\n"
b"\ttwo\r\n"
b'<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />\r\n'
b"X-After-Bad-Header: works\r\n"
b"\r\n"
b"Works"
)
def render_GET(self, request: Request) -> int:
request.startedWriting = 1
self.deferRequest(request, 0, self._delayedRender, request)
return NOT_DONE_YET
def _delayedRender(self, request: Request) -> None:
request.write(self.response)
# Clients that stop parsing headers at the bad one don't get
# Content-Length, so they need the connection to be closed to know that
# the response body is over.
close_connection(request)
class Echo(LeafResource):
def render_GET(self, request: Request) -> bytes:
assert request.content

2
tests/test_command_shell.py
View File

@ -365,7 +365,7 @@ class TestShell:
crawler.engine = MagicMock()
crawler.engine.open_spider_async = AsyncMock()
shell = Shell(crawler)
spider = Spider("test")
spider = Spider.from_crawler(crawler, "test")
await shell._open_spider(spider)
assert shell.spider is spider
assert crawler.spider is spider

2
tests/test_downloader_handler_httpx.py
View File

@ -61,6 +61,7 @@ class HttpxDownloadHandlerMixin:
class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):
handler_supports_bindaddress_meta = False
handler_bad_header_handling = "fail"
@pytest.mark.skipif(
sys.platform == "darwin",
@ -82,6 +83,7 @@ class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):
class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase):
handler_supports_bindaddress_meta = False
handler_bad_header_handling = "fail"
tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher"
@pytest.mark.skip(reason="The check is Twisted-specific")

2
tests/test_downloader_handler_twisted_ftp.py
View File

@ -212,4 +212,4 @@ class TestAnonymousFTP(TestFTPBase):
def test_not_configured_without_reactor() -> None:
crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False})
with pytest.raises(NotConfigured):
FTPDownloadHandler.from_crawler(crawler)
build_from_crawler(FTPDownloadHandler, crawler)

3
tests/test_downloader_handler_twisted_http11.py
View File

@ -11,6 +11,7 @@ from scrapy import Spider
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.utils.misc import build_from_crawler
from tests.utils.bases.download_handlers_http import (
TestHttpBase,
TestHttpProxyBase,
@ -51,7 +52,7 @@ class HTTP11DownloadHandlerMixin:
def test_not_configured_without_reactor() -> None:
crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False})
with pytest.raises(NotConfigured):
HTTP11DownloadHandler.from_crawler(crawler)
build_from_crawler(HTTP11DownloadHandler, crawler)
class TestHttp(HTTP11DownloadHandlerMixin, TestHttpBase):

3
tests/test_downloader_handler_twisted_http2.py
View File

@ -13,6 +13,7 @@ from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import DownloadFailedError, NotConfigured
from scrapy.http import Request
from scrapy.utils.misc import build_from_crawler
from tests.utils.bases.download_handlers_http import (
TestHttpProxyBase,
TestHttpsBase,
@ -66,7 +67,7 @@ def test_not_configured_without_reactor() -> None:
crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False})
with pytest.raises(NotConfigured):
H2DownloadHandler.from_crawler(crawler)
build_from_crawler(H2DownloadHandler, crawler)
class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):

3
tests/test_downloadermiddleware.py
View File

@ -14,6 +14,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.utils.decorators import coroutine_test
@ -30,7 +31,7 @@ class TestManagerBase:
async def get_mwman(self) -> AsyncGenerator[DownloaderMiddlewareManager]:
crawler = get_crawler(Spider, self.settings_dict)
crawler.spider = crawler._create_spider("foo")
mwman = DownloaderMiddlewareManager.from_crawler(crawler)
mwman = build_from_crawler(DownloaderMiddlewareManager, crawler)
crawler.engine = crawler._create_engine()
await crawler.engine.open_spider_async()
try:

21
tests/test_downloadermiddleware_cookies.py
View File

@ -10,6 +10,7 @@ from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import CookiesT, VerboseCookie
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.request import _to_verbose_cookies
from scrapy.utils.spider import DefaultSpider
@ -72,8 +73,8 @@ class TestCookiesMiddleware:
def setup_method(self):
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
self.mw = CookiesMiddleware.from_crawler(crawler)
self.redirect_middleware = RedirectMiddleware.from_crawler(crawler)
self.mw = build_from_crawler(CookiesMiddleware, crawler)
self.redirect_middleware = build_from_crawler(RedirectMiddleware, crawler)
def teardown_method(self):
del self.mw
@ -94,19 +95,19 @@ class TestCookiesMiddleware:
def test_setting_false_cookies_enabled(self):
with pytest.raises(NotConfigured):
CookiesMiddleware.from_crawler(
get_crawler(settings_dict={"COOKIES_ENABLED": False})
build_from_crawler(
CookiesMiddleware, get_crawler(settings_dict={"COOKIES_ENABLED": False})
)
def test_setting_default_cookies_enabled(self):
assert isinstance(
CookiesMiddleware.from_crawler(get_crawler()), CookiesMiddleware
build_from_crawler(CookiesMiddleware, get_crawler()), CookiesMiddleware
)
def test_setting_true_cookies_enabled(self):
assert isinstance(
CookiesMiddleware.from_crawler(
get_crawler(settings_dict={"COOKIES_ENABLED": True})
build_from_crawler(
CookiesMiddleware, get_crawler(settings_dict={"COOKIES_ENABLED": True})
),
CookiesMiddleware,
)
@ -115,7 +116,7 @@ class TestCookiesMiddleware:
self, caplog: pytest.LogCaptureFixture
) -> None:
crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True})
mw = CookiesMiddleware.from_crawler(crawler)
mw = build_from_crawler(CookiesMiddleware, crawler)
caplog.clear()
with caplog.at_level(
logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies"
@ -145,7 +146,7 @@ class TestCookiesMiddleware:
def test_debug_no_cookies(self, caplog: pytest.LogCaptureFixture) -> None:
crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True})
mw = CookiesMiddleware.from_crawler(crawler)
mw = build_from_crawler(CookiesMiddleware, crawler)
caplog.clear()
with caplog.at_level(
logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies"
@ -161,7 +162,7 @@ class TestCookiesMiddleware:
self, caplog: pytest.LogCaptureFixture
) -> None:
crawler = get_crawler(settings_dict={"COOKIES_DEBUG": False})
mw = CookiesMiddleware.from_crawler(crawler)
mw = build_from_crawler(CookiesMiddleware, crawler)
caplog.clear()
with caplog.at_level(
logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies"

3
tests/test_downloadermiddleware_defaultheaders.py
View File

@ -3,6 +3,7 @@ from __future__ import annotations
from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler
@ -13,7 +14,7 @@ def get_defaults_mw() -> tuple[dict[bytes, list[bytes]], DefaultHeadersMiddlewar
to_bytes(k): [to_bytes(v)]
for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()
}
return defaults, DefaultHeadersMiddleware.from_crawler(crawler)
return defaults, build_from_crawler(DefaultHeadersMiddleware, crawler)
def test_process_request():

3
tests/test_downloadermiddleware_downloadtimeout.py
View File

@ -5,6 +5,7 @@ from typing import Any
from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
@ -12,7 +13,7 @@ def get_request_spider_mw(settings: dict[str, Any] | None = None):
crawler = get_crawler(Spider, settings)
spider = crawler._create_spider("foo")
request = Request("http://scrapytest.org/")
return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
return request, spider, build_from_crawler(DownloadTimeoutMiddleware, crawler)
def test_default_download_timeout():

3
tests/test_downloadermiddleware_httpauth.py
View File

@ -7,6 +7,7 @@ from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
_DOMAIN_NOT_SET = object()
@ -21,7 +22,7 @@ def make_mw(
}
if domain is not _DOMAIN_NOT_SET:
settings["HTTPAUTH_DOMAIN"] = domain
return HttpAuthMiddleware.from_crawler(get_crawler(settings_dict=settings))
return build_from_crawler(HttpAuthMiddleware, get_crawler(settings_dict=settings))
# --- Spider attribute tests (deprecated) ---

3
tests/test_downloadermiddleware_httpcache.py
View File

@ -17,6 +17,7 @@ from scrapy.exceptions import IgnoreRequest
from scrapy.extensions.httpcache import DummyPolicy
from scrapy.http import HtmlResponse, Request, Response
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@ -87,7 +88,7 @@ class TestBase:
def _middleware(self, **new_settings: Any) -> Generator[HttpCacheMiddleware]:
with self._get_crawler(**new_settings) as crawler:
assert crawler.spider
mw = HttpCacheMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCacheMiddleware, crawler)
mw.spider_opened(crawler.spider)
try:
yield mw

31
tests/test_downloadermiddleware_httpcompression.py
View File

@ -18,6 +18,7 @@ from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils._compression import _DecompressionMaxSizeExceeded
from scrapy.utils.gz import gunzip
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests import tests_datadir
@ -59,7 +60,7 @@ def _skip_if_no_zstd() -> None:
class TestHttpCompression:
def setup_method(self):
self.crawler = get_crawler(Spider)
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
self.mw = build_from_crawler(HttpCompressionMiddleware, self.crawler)
assert self.crawler.stats
self.crawler.stats.open_spider()
@ -93,20 +94,22 @@ class TestHttpCompression:
def test_setting_false_compression_enabled(self):
with pytest.raises(NotConfigured):
HttpCompressionMiddleware.from_crawler(
get_crawler(settings_dict={"COMPRESSION_ENABLED": False})
build_from_crawler(
HttpCompressionMiddleware,
get_crawler(settings_dict={"COMPRESSION_ENABLED": False}),
)
def test_setting_default_compression_enabled(self):
assert isinstance(
HttpCompressionMiddleware.from_crawler(get_crawler()),
build_from_crawler(HttpCompressionMiddleware, get_crawler()),
HttpCompressionMiddleware,
)
def test_setting_true_compression_enabled(self):
assert isinstance(
HttpCompressionMiddleware.from_crawler(
get_crawler(settings_dict={"COMPRESSION_ENABLED": True})
build_from_crawler(
HttpCompressionMiddleware,
get_crawler(settings_dict={"COMPRESSION_ENABLED": True}),
),
HttpCompressionMiddleware,
)
@ -496,7 +499,7 @@ class TestHttpCompression:
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B
@ -525,7 +528,7 @@ class TestHttpCompression:
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse("bomb-gzip") # 11_511_612 B
@ -552,7 +555,7 @@ class TestHttpCompression:
crawler = get_crawler(DownloadMaxSizeSpider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
@ -584,7 +587,7 @@ class TestHttpCompression:
def _test_compression_bomb_request_meta(self, compression_id: str) -> None:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
@ -616,7 +619,7 @@ class TestHttpCompression:
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
@ -668,7 +671,7 @@ class TestHttpCompression:
crawler = get_crawler(DownloadWarnSizeSpider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
@ -721,7 +724,7 @@ class TestHttpCompression:
) -> None:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_warnsize"] = 10_000_000
@ -769,7 +772,7 @@ class TestHttpCompression:
def _get_truncated_response(self, compression_id: str) -> Response:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(compression_id)
truncated_body = response.body[: len(response.body) // 2]

3
tests/test_downloadermiddleware_httpproxy.py
View File

@ -6,6 +6,7 @@ from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
from scrapy.exceptions import NotConfigured
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
@ -20,7 +21,7 @@ class TestHttpProxyMiddleware:
def test_not_enabled(self):
crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False})
with pytest.raises(NotConfigured):
HttpProxyMiddleware.from_crawler(crawler)
build_from_crawler(HttpProxyMiddleware, crawler)
def test_no_environment_proxies(self):
os.environ.clear()

29
tests/test_downloadermiddleware_offsite.py
View File

@ -7,6 +7,7 @@ from scrapy import Request, Spider
from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
UNSET = object()
@ -31,7 +32,7 @@ UNSET = object()
def test_process_request_domain_filtering(allowed_domain, url, allowed):
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request(url)
if allowed:
@ -53,7 +54,7 @@ def test_process_request_domain_filtering(allowed_domain, url, allowed):
def test_process_request_dont_filter(value, filtered):
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
kwargs: dict[str, Any] = {}
if value is not UNSET:
@ -82,7 +83,7 @@ def test_process_request_dont_filter(value, filtered):
def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
kwargs: dict[str, Any] = {"meta": {}}
if allow_offsite is not UNSET:
@ -111,7 +112,7 @@ def test_process_request_no_allowed_domains(value):
if value is not UNSET:
kwargs["allowed_domains"] = value
crawler.spider = crawler._create_spider(name="a", **kwargs)
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request("https://example.com")
assert mw.process_request(request) is None
@ -121,7 +122,7 @@ def test_process_request_invalid_domains():
crawler = get_crawler(Spider)
allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request("https://a.example")
assert mw.process_request(request) is None
@ -150,7 +151,7 @@ def test_process_request_invalid_domains():
def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request(url)
if allowed:
@ -172,7 +173,7 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
def test_request_scheduled_dont_filter(value, filtered):
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
kwargs: dict[str, Any] = {}
if value is not UNSET:
@ -199,7 +200,7 @@ def test_request_scheduled_no_allowed_domains(value):
if value is not UNSET:
kwargs["allowed_domains"] = value
crawler.spider = crawler._create_spider(name="a", **kwargs)
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request("https://example.com")
mw.request_scheduled(request, crawler.spider)
@ -209,7 +210,7 @@ def test_request_scheduled_invalid_domains():
crawler = get_crawler(Spider)
allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request("https://a.example")
mw.request_scheduled(request, crawler.spider)
@ -222,7 +223,7 @@ def test_request_scheduled_invalid_domains():
def test_repeated_offsite_domain():
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
req1 = Request("http://other.org/1")
req2 = Request("http://other.org/2")
@ -246,7 +247,7 @@ def test_should_follow_override():
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
mw = RootOnlyOffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(RootOnlyOffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
assert mw.process_request(Request("https://example.com/1")) is None
with pytest.raises(IgnoreRequest):
@ -256,7 +257,7 @@ def test_should_follow_override():
def test_ignore_request_reason():
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request("http://other.org/1")
with pytest.raises(
@ -274,7 +275,7 @@ def test_dynamic_allowed_domains():
crawler = get_crawler(DomainSpider)
spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"])
crawler.spider = spider
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(spider)
with pytest.raises(IgnoreRequest):
@ -300,7 +301,7 @@ def test_dynamic_allowed_domains_caching():
crawler = get_crawler(DomainSpider)
spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"])
crawler.spider = spider
mw = TrackingMiddleware.from_crawler(crawler)
mw = build_from_crawler(TrackingMiddleware, crawler)
mw.spider_opened(spider)
for _ in range(3):

10
tests/test_downloadermiddleware_redirect.py
View File

@ -27,7 +27,7 @@ class TestRedirectMiddleware(TestRedirectBase):
def setup_method(self):
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
self.mw = self.mwcls.from_crawler(crawler)
self.mw = build_from_crawler(self.mwcls, crawler)
def get_response(self, request, location, status=302):
headers = {"Location": location}
@ -208,7 +208,7 @@ class TestRedirectMiddleware(TestRedirectBase):
response = Response(source_url, headers=resp_headers, status=302)
crawler = get_crawler()
referer_mw = build_from_crawler(RefererMiddleware, crawler)
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
redirect_mw._referer_spider_middleware = referer_mw
redirect_request = redirect_mw.process_response(source_request, response)
if expected_referer:
@ -223,7 +223,7 @@ class TestRedirectMiddleware(TestRedirectBase):
source_url, headers={"Referer": "http://example.com/old"}
)
response = Response(source_url, headers={"Location": redirect_url}, status=302)
redirect_mw = self.mwcls.from_crawler(get_crawler())
redirect_mw = build_from_crawler(self.mwcls, get_crawler())
redirect_mw._referer_spider_middleware = None
redirect_request = redirect_mw.process_response(source_request, response)
assert "Referer" not in redirect_request.headers
@ -352,7 +352,7 @@ class TestRedirectMiddleware(TestRedirectBase):
@pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES)
def test_redirect_schemes(url, location, target):
crawler = get_crawler(Spider)
mw = RedirectMiddleware.from_crawler(crawler)
mw = build_from_crawler(RedirectMiddleware, crawler)
request = Request(url)
response = Response(url, headers={"Location": location}, status=301)
redirect = mw.process_response(request, response)
@ -477,4 +477,4 @@ def test_warning_subclass(caplog):
def test_not_configured():
crawler = get_crawler(DefaultSpider, {"REDIRECT_ENABLED": False})
with pytest.raises(NotConfigured):
RedirectMiddleware.from_crawler(crawler)
build_from_crawler(RedirectMiddleware, crawler)

8
tests/test_downloadermiddleware_redirect_metarefresh.py
View File

@ -32,7 +32,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
def setup_method(self):
crawler = get_crawler(Spider)
self.mw = self.mwcls.from_crawler(crawler)
self.mw = build_from_crawler(self.mwcls, crawler)
def _body(
self, interval: int = 5, url: str = "http://example.org/newpage"
@ -95,7 +95,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
"""Test that Scrapy 1.x behavior remains possible"""
settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]}
crawler = get_crawler(Spider, settings)
mw = MetaRefreshMiddleware.from_crawler(crawler)
mw = build_from_crawler(MetaRefreshMiddleware, crawler)
req = Request(url="http://example.org")
body = (
"""<noscript><meta http-equiv="refresh" """
@ -134,7 +134,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
)
def test_meta_refresh_schemes(url, location, target):
crawler = get_crawler(Spider)
mw = MetaRefreshMiddleware.from_crawler(crawler)
mw = build_from_crawler(MetaRefreshMiddleware, crawler)
request = Request(url)
response = HtmlResponse(url, body=meta_refresh_body(location))
redirect = mw.process_response(request, response)
@ -169,4 +169,4 @@ def test_warning_meta_refresh_middleware(caplog):
def test_not_configured():
crawler = get_crawler(Spider, {"METAREFRESH_ENABLED": False})
with pytest.raises(NotConfigured):
MetaRefreshMiddleware.from_crawler(crawler)
build_from_crawler(MetaRefreshMiddleware, crawler)

9
tests/test_downloadermiddleware_retry.py
View File

@ -16,6 +16,7 @@ from scrapy.exceptions import (
from scrapy.http import Request, Response
from scrapy.settings.default_settings import RETRY_EXCEPTIONS
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
@ -24,7 +25,7 @@ class TestRetry:
def setup_method(self):
self.crawler = get_crawler(DefaultSpider)
self.crawler.spider = self.crawler._create_spider()
self.mw = RetryMiddleware.from_crawler(self.crawler)
self.mw = build_from_crawler(RetryMiddleware, self.crawler)
self.mw.max_retry_times = 2
def test_priority_adjust(self):
@ -94,7 +95,7 @@ class TestRetry:
DefaultSpider, settings_dict={"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"}
)
crawler.spider = crawler._create_spider()
mw = RetryMiddleware.from_crawler(crawler)
mw = build_from_crawler(RetryMiddleware, crawler)
mw.max_retry_times = 0
req = Request("http://example.com/503")
rsp = Response("http://example.com/503", body=b"", status=503)
@ -148,7 +149,7 @@ class TestRetry:
}
crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
crawler.spider = crawler._create_spider()
mw = RetryMiddleware.from_crawler(crawler)
mw = build_from_crawler(RetryMiddleware, crawler)
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
self._test_retry_exception(req, exc("foo"), mw)
@ -178,7 +179,7 @@ class TestMaxRetryTimes:
def get_middleware(self, settings: dict[str, Any] | None = None) -> RetryMiddleware:
crawler = get_crawler(DefaultSpider, settings or {})
crawler.spider = crawler._create_spider()
return RetryMiddleware.from_crawler(crawler)
return build_from_crawler(RetryMiddleware, crawler)
def test_with_settings_zero(self):
max_retry_times = 0

43
tests/test_downloadermiddleware_robotstxt.py
View File

@ -15,6 +15,7 @@ from scrapy.http.request import NO_CALLBACK
from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from tests.utils.decorators import coroutine_test
from tests.utils.robotstxt import rerp_available
@ -33,7 +34,7 @@ class TestRobotsTxtMiddleware:
self.crawler.settings = Settings()
self.crawler.settings.set("USER_AGENT", "CustomAgent")
with pytest.raises(NotConfigured):
RobotsTxtMiddleware(self.crawler)
build_from_crawler(RobotsTxtMiddleware, self.crawler)
def _get_successful_crawler(self) -> mock.MagicMock:
crawler = self.crawler
@ -60,7 +61,9 @@ Disallow: /some/randome/page.html
@coroutine_test
async def test_robotstxt(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_successful_crawler()
)
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
self.assertRobotsTxtRequested("http://site.local")
await self.assertIgnored(Request("http://site.local/admin/main"), middleware)
@ -89,7 +92,9 @@ Disallow: /some/randome/page.html
@coroutine_test
async def test_robotstxt_multiple_reqs(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_successful_crawler()
)
d1 = deferred_from_coro(
middleware.process_request(Request("http://site.local/allowed1"))
)
@ -101,20 +106,26 @@ Disallow: /some/randome/page.html
@pytest.mark.only_asyncio
@coroutine_test
async def test_robotstxt_multiple_reqs_asyncio(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_successful_crawler()
)
c1 = middleware.process_request(Request("http://site.local/allowed1"))
c2 = middleware.process_request(Request("http://site.local/allowed2"))
await asyncio.gather(c1, c2)
@coroutine_test
async def test_robotstxt_ready_parser(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_successful_crawler()
)
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
@coroutine_test
async def test_robotstxt_meta(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_successful_crawler()
)
meta = {"dont_obey_robotstxt": True}
await self.assertNotIgnored(
Request("http://site.local/allowed", meta=meta), middleware
@ -144,7 +155,9 @@ Disallow: /some/randome/page.html
@coroutine_test
async def test_robotstxt_garbage(self):
# garbage response should be discarded, equal 'allow all'
middleware = RobotsTxtMiddleware(self._get_garbage_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_garbage_crawler()
)
await self.assertNotIgnored(Request("http://site.local"), middleware)
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)
@ -166,7 +179,9 @@ Disallow: /some/randome/page.html
@coroutine_test
async def test_robotstxt_empty_response(self):
# empty response should equal 'allow all'
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_emptybody_crawler()
)
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)
await self.assertNotIgnored(Request("http://site.local/static/"), middleware)
@ -183,7 +198,7 @@ Disallow: /some/randome/page.html
self.crawler.engine.download_async.side_effect = return_failure
middleware = RobotsTxtMiddleware(self.crawler)
middleware = build_from_crawler(RobotsTxtMiddleware, self.crawler)
await middleware.process_request(Request("http://site.local"))
assert "Robotstxt address not found" in caplog.text
@ -197,7 +212,7 @@ Disallow: /some/randome/page.html
self.crawler.engine.download_async.side_effect = immediate_failure
middleware = RobotsTxtMiddleware(self.crawler)
middleware = build_from_crawler(RobotsTxtMiddleware, self.crawler)
await self.assertNotIgnored(Request("http://site.local"), middleware)
@coroutine_test
@ -211,7 +226,7 @@ Disallow: /some/randome/page.html
self.crawler.engine.download_async.side_effect = ignore_request
middleware = RobotsTxtMiddleware(self.crawler)
middleware = build_from_crawler(RobotsTxtMiddleware, self.crawler)
with mock.patch(
"scrapy.downloadermiddlewares.robotstxt.logger"
) as mw_module_logger:
@ -224,14 +239,16 @@ Disallow: /some/randome/page.html
crawler = self._get_successful_crawler()
crawler.settings.set("ROBOTSTXT_USER_AGENT", "Examplebot")
crawler.settings.set("USER_AGENT", "Mozilla/5.0 (X11; Linux x86_64)")
middleware = RobotsTxtMiddleware(crawler)
middleware = build_from_crawler(RobotsTxtMiddleware, crawler)
rp = mock.MagicMock(return_value=True)
middleware.process_request_2(rp, Request("http://site.local/allowed"))
rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot")
@coroutine_test
async def test_robotstxt_local_file(self):
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_emptybody_crawler()
)
middleware.process_request_2 = mock.MagicMock() # type: ignore[method-assign]
await middleware.process_request(Request("data:text/plain,Hello World data"))

5
tests/test_downloadermiddleware_stats.py
View File

@ -6,6 +6,7 @@ from scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_size
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
@ -17,7 +18,7 @@ class TestDownloaderStats:
def setup_method(self) -> None:
self.crawler = get_crawler(Spider)
assert self.crawler.stats is not None
self.mw = DownloaderStats(self.crawler.stats)
self.mw = build_from_crawler(DownloaderStats, self.crawler)
self.crawler.stats.open_spider()
@ -49,7 +50,7 @@ class TestDownloaderStats:
def test_from_crawler_not_configured(self) -> None:
crawler = get_crawler(Spider, {"DOWNLOADER_STATS": False})
with pytest.raises(NotConfigured):
DownloaderStats.from_crawler(crawler)
build_from_crawler(DownloaderStats, crawler)
def teardown_method(self) -> None:
assert self.crawler.stats is not None

3
tests/test_downloadermiddleware_useragent.py
View File

@ -3,6 +3,7 @@ from __future__ import annotations
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
@ -11,7 +12,7 @@ def get_spider_and_mw(
) -> tuple[Spider, UserAgentMiddleware]:
crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})
spider = crawler._create_spider("foo")
return spider, UserAgentMiddleware.from_crawler(crawler)
return spider, build_from_crawler(UserAgentMiddleware, crawler)
def test_default_agent():

9
tests/test_dupefilters.py
View File

@ -14,6 +14,7 @@ from scrapy.core.scheduler import Scheduler
from scrapy.dupefilters import BaseDupeFilter, RFPDupeFilter
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
@ -30,7 +31,7 @@ def _get_dupefilter(
) -> BaseDupeFilter:
if crawler is None:
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
scheduler = build_from_crawler(Scheduler, crawler)
dupefilter = scheduler.df
if open_:
dupefilter.open()
@ -56,7 +57,7 @@ class TestRFPDupeFilter:
"DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
scheduler = build_from_crawler(Scheduler, crawler)
assert scheduler.df.debug
assert scheduler.df.method == "from_crawler"
@ -65,7 +66,7 @@ class TestRFPDupeFilter:
"DUPEFILTER_CLASS": DirectDupeFilter,
}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
scheduler = build_from_crawler(Scheduler, crawler)
assert scheduler.df.method == "n/a"
def test_filter(self):
@ -145,7 +146,7 @@ class TestRFPDupeFilter:
path = tempfile.mkdtemp()
crawler = get_crawler(settings_dict={"JOBDIR": path})
try:
scheduler = Scheduler.from_crawler(crawler)
scheduler = build_from_crawler(Scheduler, crawler)
df = scheduler.df
df.open()
df.request_seen(r1)

3
tests/test_engine.py
View File

@ -16,6 +16,7 @@ from scrapy.exceptions import CloseSpider, IgnoreRequest
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.defer import _schedule_coro, deferred_from_coro
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.bases.engine import TestEngineBase
@ -215,7 +216,7 @@ async def test_request_scheduled_signal():
crawler = get_crawler(MySpider)
engine = ExecutionEngine(crawler, lambda _: None)
scheduler = TestScheduler() # type: ignore[abstract]
scheduler = build_from_crawler(TestScheduler, crawler)
async def start() -> AsyncIterator[Any]:
return

4
tests/test_exporters.py
View File

@ -578,7 +578,7 @@ class TestJsonLinesItemExporter(TestBaseItemExporter):
self.ie.finish_exporting()
del self.ie # See the first “del self.ie” in this file for context.
exported = json.loads(to_unicode(self.output.getvalue()))
item["time"] = str(item["time"])
item["time"] = item["time"].isoformat()
assert exported == item
@ -661,7 +661,7 @@ class TestJsonItemExporter(TestJsonLinesItemExporter):
self.ie.finish_exporting()
del self.ie # See the first “del self.ie” in this file for context.
exported = json.loads(to_unicode(self.output.getvalue()))
item["time"] = str(item["time"])
item["time"] = item["time"].isoformat()
assert exported == [item]

11
tests/test_extension_debug.py
View File

@ -12,6 +12,7 @@ import pytest
from scrapy.extensions.debug import Debugger, StackTraceDump
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
@ -48,7 +49,7 @@ class SignalSpider(Spider):
)
def test_stacktracedump_installs_signal_handlers() -> None:
crawler = get_crawler()
ext = StackTraceDump.from_crawler(crawler)
ext = build_from_crawler(StackTraceDump, crawler)
assert signal.getsignal(signal.SIGUSR2) == ext.dump_stacktrace # pylint: disable=comparison-with-callable
assert signal.getsignal(signal.SIGQUIT) == ext.dump_stacktrace # pylint: disable=comparison-with-callable
@ -58,15 +59,15 @@ def test_stacktracedump_works_without_signal_support(
) -> None:
# simulate win32 platforms, which don't support SIGUSR signals
monkeypatch.delattr(signal, "SIGUSR2", raising=False)
ext = StackTraceDump.from_crawler(get_crawler())
ext = build_from_crawler(StackTraceDump, get_crawler())
assert isinstance(ext, StackTraceDump)
def test_stacktracedump_dump_stacktrace(caplog: pytest.LogCaptureFixture) -> None:
crawler = get_crawler()
crawler.engine = mock.Mock()
ext = StackTraceDump.from_crawler(crawler)
spider = DefaultSpider()
ext = build_from_crawler(StackTraceDump, crawler)
spider = DefaultSpider.from_crawler(crawler)
with caplog.at_level(logging.INFO, logger="scrapy.extensions.debug"):
ext.dump_stacktrace(0, None)
for r in caplog.records:
@ -82,7 +83,7 @@ def test_stacktracedump_dump_stacktrace(caplog: pytest.LogCaptureFixture) -> Non
def test_stacktracedump_thread_stacks() -> None:
ext = StackTraceDump.from_crawler(get_crawler())
ext = build_from_crawler(StackTraceDump, get_crawler())
stop = threading.Event()
thread = threading.Thread(target=stop.wait, name="dump-test-thread")
thread.start()

7
tests/test_extension_memdebug.py
View File

@ -6,6 +6,7 @@ import pytest
from scrapy.exceptions import NotConfigured
from scrapy.extensions.memdebug import MemoryDebugger
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from scrapy.utils.trackref import object_ref
@ -14,12 +15,12 @@ from tests.utils.decorators import coroutine_test
def test_disabled_by_default() -> None:
with pytest.raises(NotConfigured):
MemoryDebugger.from_crawler(get_crawler())
build_from_crawler(MemoryDebugger, get_crawler())
def test_spider_closed_sets_stats() -> None:
crawler = get_crawler(settings_dict={"MEMDEBUG_ENABLED": True})
ext = MemoryDebugger.from_crawler(crawler)
ext = build_from_crawler(MemoryDebugger, crawler)
class TrackedObject(object_ref):
pass
@ -30,7 +31,7 @@ def test_spider_closed_sets_stats() -> None:
tracked = [TrackedObject(), TrackedObject()]
CollectedObject()
ext.spider_closed(DefaultSpider(), "finished")
ext.spider_closed(DefaultSpider.from_crawler(crawler), "finished")
assert crawler.stats
assert crawler.stats.get_value("memdebug/gc_garbage_count") == len(gc.garbage)

3
tests/test_extension_memusage.py
View File

@ -12,6 +12,7 @@ from scrapy.exceptions import NotConfigured
from scrapy.extensions import memusage as memusage_mod
from scrapy.extensions.memusage import MemoryUsage
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils import OneShotLoop
from tests.utils.cmdline import proc
@ -60,7 +61,7 @@ def test_memusage_disabled() -> None:
"MEMUSAGE_ENABLED": False,
}
with pytest.raises(NotConfigured):
MemoryUsage.from_crawler(get_crawler(settings_dict=settings))
build_from_crawler(MemoryUsage, get_crawler(settings_dict=settings))
def test_memusage_limit_stops_crawler_without_spider(mockserver: MockServer) -> None:

3
tests/test_extension_periodic_log.py
View File

@ -9,6 +9,7 @@ import pytest
from scrapy.exceptions import NotConfigured
from scrapy.extensions.periodic_log import PeriodicLog
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from .spiders import MetaSpider
@ -72,7 +73,7 @@ class CustomPeriodicLog(PeriodicLog):
def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog:
crawler = get_crawler(MetaSpider, settings)
return CustomPeriodicLog.from_crawler(crawler)
return build_from_crawler(CustomPeriodicLog, crawler)
class TestPeriodicLog:

7
tests/test_extension_statsmailer.py
View File

@ -7,6 +7,7 @@ from scrapy import signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.signalmanager import SignalManager
from scrapy.statscollectors import StatsCollector
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
with warnings.catch_warnings():
@ -43,7 +44,7 @@ def test_from_crawler_without_recipients_raises_notconfigured():
crawler.stats = MagicMock()
with pytest.raises(NotConfigured):
statsmailer.StatsMailer.from_crawler(crawler)
build_from_crawler(statsmailer.StatsMailer, crawler)
def test_from_crawler_with_recipients_initializes_extension(dummy_stats, monkeypatch):
@ -55,7 +56,7 @@ def test_from_crawler_with_recipients_initializes_extension(dummy_stats, monkeyp
mailer = MagicMock(spec=MailSender)
monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer)
ext = statsmailer.StatsMailer.from_crawler(crawler)
ext = build_from_crawler(statsmailer.StatsMailer, crawler)
assert isinstance(ext, statsmailer.StatsMailer)
assert ext.recipients == ["test@example.com"]
@ -71,7 +72,7 @@ def test_from_crawler_connects_spider_closed_signal(dummy_stats, monkeypatch):
mailer = MagicMock(spec=MailSender)
monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer)
statsmailer.StatsMailer.from_crawler(crawler)
build_from_crawler(statsmailer.StatsMailer, crawler)
connected = crawler.signals.send_catch_log(
signals.spider_closed, spider=DefaultSpider(name="dummy")

5
tests/test_extension_telnet.py
View File

@ -11,6 +11,7 @@ from twisted.cred import credentials
from scrapy import Spider
from scrapy.extensions.telnet import TelnetConsole, update_telnet_vars
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
@ -39,7 +40,7 @@ def _get_console_and_portal(
settings: dict[str, Any] | None = None,
) -> Generator[tuple[TelnetConsole, Any]]:
crawler = _get_crawler(settings_dict=settings)
console = TelnetConsole(crawler)
console = build_from_crawler(TelnetConsole, crawler)
# This function has some side effects we don't need for this test
console._get_telnet_vars = dict # type: ignore[method-assign]
@ -87,7 +88,7 @@ async def test_custom_credentials() -> None:
def test_invalid_reversed_portrange() -> None:
settings = {"TELNETCONSOLE_PORT": [2, 1]}
console = TelnetConsole(_get_crawler(settings_dict=settings))
console = build_from_crawler(TelnetConsole, _get_crawler(settings_dict=settings))
with pytest.raises(ValueError, match=r"invalid portrange: \[2, 1\]"):
console.start_listening()

18
tests/test_extension_throttle.py
View File

@ -81,7 +81,7 @@ def test_mindelay_definition(setting, expected):
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
_mock_downloader(crawler)
at._spider_opened(DefaultSpider())
at._spider_opened(DefaultSpider.from_crawler(crawler))
assert at.mindelay == expected
@ -99,7 +99,7 @@ def test_maxdelay_definition(value, expected):
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
_mock_downloader(crawler)
at._spider_opened(DefaultSpider())
at._spider_opened(DefaultSpider.from_crawler(crawler))
assert at.maxdelay == expected
@ -133,7 +133,7 @@ def test_startdelay_definition(min_setting, start_setting, expected):
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
at._spider_opened(DefaultSpider())
at._spider_opened(DefaultSpider.from_crawler(crawler))
assert downloader._delay == expected
@ -159,7 +159,7 @@ def test_skipped(meta, slot):
crawler = get_crawler()
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
spider = DefaultSpider.from_crawler(crawler)
at._spider_opened(spider)
request = Request("https://example.com", meta=meta)
@ -187,7 +187,7 @@ def test_adjustment(download_latency, target_concurrency, slot_delay, expected):
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
spider = DefaultSpider.from_crawler(crawler)
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
@ -221,7 +221,7 @@ def test_adjustment_limits(mindelay, maxdelay, expected):
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
spider = DefaultSpider.from_crawler(crawler)
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
@ -251,7 +251,7 @@ def test_adjustment_bad_response(
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
spider = DefaultSpider.from_crawler(crawler)
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
@ -271,7 +271,7 @@ def test_debug(caplog):
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
spider = DefaultSpider.from_crawler(crawler)
at._spider_opened(spider)
meta = {"download_latency": 1.0, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
@ -299,7 +299,7 @@ def test_debug_disabled(caplog):
crawler = get_crawler()
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
spider = DefaultSpider.from_crawler(crawler)
at._spider_opened(spider)
meta = {"download_latency": 1.0, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)

16
tests/test_feedexport.py
View File

@ -27,6 +27,7 @@ from scrapy.extensions.feedexport import (
ItemFilter,
apply_uri_params,
)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
from tests.spiders import ItemSpider
@ -1262,9 +1263,8 @@ class TestFeedExporterSignals:
feed_slot_signal_handler: Callable[[Any], Awaitable[None] | None],
) -> None:
crawler = get_crawler(settings_dict=self.settings)
feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
spider.crawler = crawler
feed_exporter = build_from_crawler(FeedExporter, crawler)
spider = scrapy.Spider.from_crawler(crawler, "default")
crawler.signals.connect(
feed_exporter_signal_handler,
signal=signals.feed_exporter_closed,
@ -1318,7 +1318,7 @@ class TestFeedExportInit:
}
crawler = get_crawler(settings_dict=settings)
with pytest.raises(NotConfigured):
FeedExporter.from_crawler(crawler)
build_from_crawler(FeedExporter, crawler)
def test_disabled_storage(self, caplog: pytest.LogCaptureFixture):
class DisabledFeedStorage:
@ -1333,7 +1333,7 @@ class TestFeedExportInit:
}
crawler = get_crawler(settings_dict=settings)
with caplog.at_level(logging.ERROR), pytest.raises(NotConfigured):
FeedExporter.from_crawler(crawler)
build_from_crawler(FeedExporter, crawler)
assert (
"Disabled feed storage scheme: disabled. Reason: not today" in caplog.text
)
@ -1348,7 +1348,7 @@ class TestFeedExportInit:
}
crawler = get_crawler(settings_dict=settings)
with pytest.raises(NotConfigured):
FeedExporter.from_crawler(crawler)
build_from_crawler(FeedExporter, crawler)
def test_absolute_pathlib_as_uri(self):
with tempfile.NamedTemporaryFile(suffix="json") as tmp:
@ -1360,7 +1360,7 @@ class TestFeedExportInit:
},
}
crawler = get_crawler(settings_dict=settings)
exporter = FeedExporter.from_crawler(crawler)
exporter = build_from_crawler(FeedExporter, crawler)
assert isinstance(exporter, FeedExporter)
def test_relative_pathlib_as_uri(self):
@ -1372,7 +1372,7 @@ class TestFeedExportInit:
},
}
crawler = get_crawler(settings_dict=settings)
exporter = FeedExporter.from_crawler(crawler)
exporter = build_from_crawler(FeedExporter, crawler)
assert isinstance(exporter, FeedExporter)

3
tests/test_feedexport_batch.py
View File

@ -18,6 +18,7 @@ from scrapy import Spider
from scrapy.exceptions import NotConfigured
from scrapy.extensions.feedexport import FeedExporter, S3FeedStorage
from scrapy.settings import Settings
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
from tests.spiders import ItemSpider
@ -261,7 +262,7 @@ class TestBatchDeliveries(TestFeedExportBase):
}
crawler = get_crawler(settings_dict=settings)
with pytest.raises(NotConfigured):
FeedExporter(crawler)
build_from_crawler(FeedExporter, crawler)
@coroutine_test
async def test_export_no_items_not_store_empty(self):

58
tests/test_feedexport_storages.py
View File

@ -7,6 +7,7 @@ import sys
import tempfile
from io import BytesIO
from pathlib import Path
from ssl import SSLCertVerificationError
from typing import IO, Any
from unittest import mock
from urllib.parse import quote
@ -25,6 +26,7 @@ from scrapy.extensions.feedexport import (
StdoutFeedStorage,
)
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
from tests.utils.cloud import mock_google_cloud_storage
@ -112,7 +114,8 @@ class TestFTPFeedStorage:
settings: dict[str, Any] | None = None,
) -> None:
crawler = get_crawler(settings_dict=settings or {})
storage = FTPFeedStorage.from_crawler(
storage = build_from_crawler(
FTPFeedStorage,
crawler,
uri,
feed_options=feed_options,
@ -169,6 +172,24 @@ class TestFTPFeedStorage:
await self._store(url, b"bar", settings=settings)
self._assert_stored(ftp_server.path / filename, b"bar")
@coroutine_test
async def test_tls(self, monkeypatch):
monkeypatch.setenv(
"SSL_CERT_FILE", str(Path(__file__).parent / "keys" / "localhost.crt")
)
with MockFTPServer(tls=True) as ftp_server:
filename = "file"
await self._store(ftp_server.url(filename), b"foo")
self._assert_stored(ftp_server.path / filename, b"foo")
@coroutine_test
async def test_tls_untrusted_certificate(self):
with (
MockFTPServer(tls=True) as ftp_server,
pytest.raises(SSLCertVerificationError),
):
await self._store(ftp_server.url("file"), b"foo")
def test_uri_auth_quote(self):
# RFC3986: 3.2.1. User Information
pw_quoted = quote(string.punctuation, safe="")
@ -230,7 +251,8 @@ class TestS3FeedStorage:
}
crawler = get_crawler(settings_dict=aws_credentials)
# Instantiate with crawler
storage = S3FeedStorage.from_crawler(
storage = build_from_crawler(
S3FeedStorage,
crawler,
"s3://mybucket/export.csv",
)
@ -265,7 +287,7 @@ class TestS3FeedStorage:
crawler = get_crawler(settings_dict=settings)
bucket = "mybucket"
key = "export.csv"
storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}")
storage = build_from_crawler(S3FeedStorage, crawler, f"s3://{bucket}/{key}")
file = mock.MagicMock()
@ -319,7 +341,8 @@ class TestS3FeedStorage:
"AWS_SECRET_ACCESS_KEY": "secret_key",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
storage = build_from_crawler(
S3FeedStorage,
crawler,
"s3://mybucket/export.csv",
)
@ -333,7 +356,8 @@ class TestS3FeedStorage:
"AWS_SECRET_ACCESS_KEY": "secret_key",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
storage = build_from_crawler(
S3FeedStorage,
crawler,
"s3://mybucket/export.csv",
)
@ -347,7 +371,8 @@ class TestS3FeedStorage:
"AWS_SECRET_ACCESS_KEY": "secret_key",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
storage = build_from_crawler(
S3FeedStorage,
crawler,
"s3://mybucket/export.csv",
)
@ -362,7 +387,8 @@ class TestS3FeedStorage:
"FEED_STORAGE_S3_ACL": "custom-acl",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
storage = build_from_crawler(
S3FeedStorage,
crawler,
"s3://mybucket/export.csv",
)
@ -377,7 +403,7 @@ class TestS3FeedStorage:
"AWS_ENDPOINT_URL": "https://example.com",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")
storage = build_from_crawler(S3FeedStorage, crawler, "s3://mybucket/export.csv")
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.endpoint_url == "https://example.com"
@ -390,7 +416,7 @@ class TestS3FeedStorage:
"AWS_REGION_NAME": region_name,
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")
storage = build_from_crawler(S3FeedStorage, crawler, "s3://mybucket/export.csv")
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.region_name == region_name
@ -426,7 +452,7 @@ class TestS3FeedStorage:
self, settings: dict[str, Any], expected: int
) -> None:
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")
storage = build_from_crawler(S3FeedStorage, crawler, "s3://mybucket/export.csv")
assert storage.max_pool_connections == expected
config: Any = storage.s3_client.meta.config
assert config.max_pool_connections == expected
@ -488,7 +514,9 @@ class TestGCSFeedStorage:
settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"}
crawler = get_crawler(settings_dict=settings)
storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
storage = build_from_crawler(
GCSFeedStorage, crawler, "gs://mybucket/export.csv"
)
assert storage.project_id == "123"
assert storage.acl == "publicRead"
assert storage.bucket_name == "mybucket"
@ -499,12 +527,16 @@ class TestGCSFeedStorage:
settings: dict[str, Any] = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""}
crawler = get_crawler(settings_dict=settings)
storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
storage = build_from_crawler(
GCSFeedStorage, crawler, "gs://mybucket/export.csv"
)
assert storage.acl is None
settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": None}
crawler = get_crawler(settings_dict=settings)
storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
storage = build_from_crawler(
GCSFeedStorage, crawler, "gs://mybucket/export.csv"
)
assert storage.acl is None
@coroutine_test

15
tests/test_feedexport_uri_params.py
View File

@ -49,8 +49,7 @@ class TestURIParams(ABC):
uri="file:///tmp/%(name)s",
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
spider = scrapy.Spider.from_crawler(crawler, self.spider_name)
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
@ -67,8 +66,7 @@ class TestURIParams(ABC):
uri_params=uri_params,
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
spider = scrapy.Spider.from_crawler(crawler, self.spider_name)
feed_exporter.open_spider(spider)
@ -83,8 +81,7 @@ class TestURIParams(ABC):
uri_params=uri_params,
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
spider = scrapy.Spider.from_crawler(crawler, self.spider_name)
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
@ -100,8 +97,7 @@ class TestURIParams(ABC):
uri_params=uri_params,
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
spider = scrapy.Spider.from_crawler(crawler, self.spider_name)
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)
@ -117,8 +113,7 @@ class TestURIParams(ABC):
uri_params=uri_params,
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
spider = scrapy.Spider.from_crawler(crawler, self.spider_name)
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)

16
tests/test_logformatter.py
View File

@ -11,6 +11,7 @@ from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.logformatter import LogFormatter
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.spiders import ItemSpider
from tests.utils.decorators import coroutine_test
@ -28,9 +29,9 @@ class CustomItem(Item):
class TestLogFormatter:
def setup_method(self):
self.formatter = LogFormatter()
self.spider = Spider("default")
self.spider.crawler = get_crawler()
crawler = get_crawler()
self.formatter = build_from_crawler(LogFormatter, crawler)
self.spider = Spider.from_crawler(crawler, "default")
def test_crawled_without_referer(self):
req = Request("http://www.example.com")
@ -75,8 +76,7 @@ class TestLogFormatter:
item = {}
exception = DropItem("Test drop")
response = Response("http://www.example.com")
spider = Spider("foo")
spider.crawler = get_crawler(Spider)
spider = Spider.from_crawler(get_crawler(Spider), "foo")
logkws = self.formatter.dropped(item, exception, response, spider)
assert logkws["level"] == logging.WARNING
@ -198,9 +198,9 @@ class LogFormatterSubclass(LogFormatter):
class TestLogformatterSubclass(TestLogFormatter):
def setup_method(self):
self.formatter = LogFormatterSubclass()
self.spider = Spider("default")
self.spider.crawler = get_crawler(Spider)
crawler = get_crawler(Spider)
self.formatter = build_from_crawler(LogFormatterSubclass, crawler)
self.spider = Spider.from_crawler(crawler, "default")
def test_crawled_without_referer(self):
req = Request("http://www.example.com")

7
tests/test_logstats.py
View File

@ -5,6 +5,7 @@ from datetime import datetime
import pytest
from scrapy.extensions.logstats import LogStats
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
from tests.utils.decorators import coroutine_test
@ -22,7 +23,7 @@ class TestLogStats:
@coroutine_test
async def test_stats_calculations(self) -> None:
logstats = LogStats.from_crawler(self.crawler)
logstats = build_from_crawler(LogStats, self.crawler)
with pytest.raises(AttributeError):
logstats.pagesprev
@ -63,14 +64,14 @@ class TestLogStats:
"""The stat values should be None since the start and finish time are
not available.
"""
logstats = LogStats.from_crawler(self.crawler)
logstats = build_from_crawler(LogStats, self.crawler)
logstats.spider_closed(self.spider, "test reason")
assert self.stats.get_value("responses_per_minute") is None
assert self.stats.get_value("items_per_minute") is None
def test_stats_calculation_no_elapsed_time(self) -> None:
"""The stat values should be None since the elapsed time is 0."""
logstats = LogStats.from_crawler(self.crawler)
logstats = build_from_crawler(LogStats, self.crawler)
self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))
self.stats.set_value("finish_time", datetime.fromtimestamp(1655100172))
logstats.spider_closed(self.spider, "test reason")

3
tests/test_middleware.py
View File

@ -7,6 +7,7 @@ import pytest
from scrapy import Spider
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.middleware import MiddlewareManager
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@ -93,7 +94,7 @@ def test_enabled(crawler: Crawler) -> None:
def test_enabled_from_settings(crawler: Crawler) -> None:
crawler = get_crawler()
mwman = MyMiddlewareManager.from_crawler(crawler)
mwman = build_from_crawler(MyMiddlewareManager, crawler)
classes = [x.__class__ for x in mwman.middlewares]
assert classes == [M1, M3]
assert mwman.crawler == crawler

77
tests/test_pipeline_files.py
View File

@ -25,21 +25,22 @@ from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.pipelines import files
from scrapy.pipelines.files import (
FileException,
FilesPipeline,
FSFilesStore,
FTPFilesStore,
GCSFilesStore,
S3FilesStore,
)
from scrapy.pipelines.media import _MediaRequestFiltered
from scrapy.pipelines.media import FileException, _MediaRequestFiltered
from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
@ -108,7 +109,7 @@ class TestFilesPipeline:
)
crawler.spider = crawler._create_spider()
crawler.engine = MagicMock(download_async=mocked_download_func)
pipeline = pipeline_cls.from_crawler(crawler)
pipeline = build_from_crawler(pipeline_cls, crawler)
pipeline.open_spider()
return pipeline
@ -448,8 +449,8 @@ class TestFilesPipeline:
def file_path(self, request, response=None, info=None, item=None) -> str:
return f"full/{item.get('path')}"
file_path = CustomFilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": self.tempdir})
file_path = build_from_crawler(
CustomFilesPipeline, get_crawler(None, {"FILES_STORE": self.tempdir})
).file_path
item = {"path": "path-to-store-file"}
request = Request("http://example.com")
@ -572,8 +573,8 @@ class TestFilesPipeline:
],
)
def test_rejects_non_list_file_urls(self, tmp_path, bad_type):
pipeline = FilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": str(tmp_path)})
pipeline = build_from_crawler(
FilesPipeline, get_crawler(None, {"FILES_STORE": str(tmp_path)})
)
item = ItemWithFiles()
item["file_urls"] = bad_type
@ -591,8 +592,8 @@ class TestFilesPipelineFieldsMixin(ABC):
def test_item_fields_default(self, tmp_path):
url = "http://www.example.com/files/1.txt"
item = self.item_class(name="item1", file_urls=[url])
pipeline = FilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": tmp_path})
pipeline = build_from_crawler(
FilesPipeline, get_crawler(None, {"FILES_STORE": tmp_path})
)
requests = list(pipeline.get_media_requests(item, None)) # type: ignore[arg-type]
assert requests[0].url == url
@ -605,7 +606,8 @@ class TestFilesPipelineFieldsMixin(ABC):
def test_item_fields_override_settings(self, tmp_path):
url = "http://www.example.com/files/1.txt"
item = self.item_class(name="item1", custom_file_urls=[url])
pipeline = FilesPipeline.from_crawler(
pipeline = build_from_crawler(
FilesPipeline,
get_crawler(
None,
{
@ -613,7 +615,7 @@ class TestFilesPipelineFieldsMixin(ABC):
"FILES_URLS_FIELD": "custom_file_urls",
"FILES_RESULT_FIELD": "custom_files",
},
)
),
)
requests = list(pipeline.get_media_requests(item, None)) # type: ignore[arg-type]
assert requests[0].url == url
@ -718,10 +720,12 @@ class TestFilesPipelineCustomSettings:
different settings.
"""
custom_settings = self._generate_fake_settings(tmp_path)
another_pipeline = FilesPipeline.from_crawler(
get_crawler(None, custom_settings)
another_pipeline = build_from_crawler(
FilesPipeline, get_crawler(None, custom_settings)
)
one_pipeline = build_from_crawler(
FilesPipeline, get_crawler(None, {"FILES_STORE": tmp_path})
)
one_pipeline = FilesPipeline(tmp_path, crawler=get_crawler(None))
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
default_value = self.default_cls_settings[pipe_attr]
assert getattr(one_pipeline, pipe_attr) == default_value
@ -734,7 +738,9 @@ class TestFilesPipelineCustomSettings:
If subclasses override class attributes and there are no special settings those values should be kept.
"""
pipe_cls = self._generate_fake_pipeline()
pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": tmp_path}))
pipe = build_from_crawler(
pipe_cls, get_crawler(None, {"FILES_STORE": tmp_path})
)
for pipe_attr, _, pipe_ins_attr in self.file_cls_attr_settings_map:
custom_value = getattr(pipe, pipe_ins_attr)
assert custom_value != self.default_cls_settings[pipe_attr]
@ -747,7 +753,7 @@ class TestFilesPipelineCustomSettings:
"""
pipeline_cls = self._generate_fake_pipeline()
settings = self._generate_fake_settings(tmp_path)
pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
pipeline = build_from_crawler(pipeline_cls, get_crawler(None, settings))
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
value = getattr(pipeline, pipe_ins_attr)
setting_value = settings.get(settings_attr)
@ -763,8 +769,8 @@ class TestFilesPipelineCustomSettings:
class UserDefinedFilesPipeline(FilesPipeline):
pass
user_pipeline = UserDefinedFilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": tmp_path})
user_pipeline = build_from_crawler(
UserDefinedFilesPipeline, get_crawler(None, {"FILES_STORE": tmp_path})
)
for pipe_attr, _, pipe_ins_attr in self.file_cls_attr_settings_map:
# Values from settings for custom pipeline should be set on pipeline instance.
@ -782,8 +788,8 @@ class TestFilesPipelineCustomSettings:
prefix = UserDefinedFilesPipeline.__name__.upper()
settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = UserDefinedFilesPipeline.from_crawler(
get_crawler(None, settings)
user_pipeline = build_from_crawler(
UserDefinedFilesPipeline, get_crawler(None, settings)
)
for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
# Values from settings for custom pipeline should be set on pipeline instance.
@ -799,7 +805,7 @@ class TestFilesPipelineCustomSettings:
pipeline_cls = self._generate_fake_pipeline()
prefix = pipeline_cls.__name__.upper()
settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
user_pipeline = build_from_crawler(pipeline_cls, get_crawler(None, settings))
for (
pipe_cls_attr,
settings_attr,
@ -814,8 +820,8 @@ class TestFilesPipelineCustomSettings:
DEFAULT_FILES_RESULT_FIELD = "this"
DEFAULT_FILES_URLS_FIELD = "that"
pipeline = UserDefinedFilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": tmp_path})
pipeline = build_from_crawler(
UserDefinedFilesPipeline, get_crawler(None, {"FILES_STORE": tmp_path})
)
assert (
pipeline.files_result_field
@ -836,7 +842,7 @@ class TestFilesPipelineCustomSettings:
class UserPipe(FilesPipeline):
pass
pipeline_cls = UserPipe.from_crawler(get_crawler(None, settings))
pipeline_cls = build_from_crawler(UserPipe, get_crawler(None, settings))
for _, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
expected_value = settings.get(settings_attr)
@ -847,8 +853,9 @@ class TestFilesPipelineCustomSettings:
def file_path(self, request, response=None, info=None, *, item=None) -> str:
return str(Path("subdir") / Path(request.url).name)
pipeline = CustomFilesPipelineWithPathLikeDir.from_crawler(
get_crawler(None, {"FILES_STORE": tmp_path})
pipeline = build_from_crawler(
CustomFilesPipelineWithPathLikeDir,
get_crawler(None, {"FILES_STORE": tmp_path}),
)
request = Request("http://example.com/image01.jpg")
assert pipeline.file_path(request) == str(Path("subdir/image01.jpg"))
@ -1054,7 +1061,7 @@ class TestS3FilesStore:
crawler = get_crawler(
settings_dict={"FILES_STORE": "s3://mybucket/prefix/", **settings}
)
store = FilesPipeline.from_crawler(crawler).store
store = build_from_crawler(FilesPipeline, crawler).store
assert isinstance(store, S3FilesStore)
config: Any = store.s3_client.meta.config
assert config.max_pool_connections == expected
@ -1312,7 +1319,7 @@ class TestBuildFromCrawler:
class Pipeline(FilesPipeline):
pass
pipe = Pipeline.from_crawler(self.crawler)
pipe = build_from_crawler(Pipeline, self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert pipe.store
@ -1329,7 +1336,7 @@ class TestBuildFromCrawler:
o._from_crawler_called = True
return o
pipe = Pipeline.from_crawler(self.crawler)
pipe = build_from_crawler(Pipeline, self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert pipe.store
@ -1344,4 +1351,12 @@ def test_files_pipeline_raises_notconfigured_when_files_store_invalid(store):
crawler = get_crawler(settings_dict=dict(settings))
with pytest.raises(NotConfigured):
FilesPipeline.from_crawler(crawler)
build_from_crawler(FilesPipeline, crawler)
def test_file_exception_deprecated_import():
with pytest.warns(ScrapyDeprecationWarning, match="FileException"):
assert files.FileException is FileException
with pytest.raises(AttributeError):
files.nonexistent

56
tests/test_pipeline_images.py
View File

@ -21,6 +21,7 @@ from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.pipelines.files import GCSFilesStore, S3FilesStore, _checksum
from scrapy.pipelines.images import ImageException, ImagesPipeline
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
from tests.utils.media_pipelines import DUMMY_SPIDER_INFO
@ -41,8 +42,8 @@ else:
class TestImagesPipeline:
def setup_method(self):
self.tempdir = mkdtemp()
crawler = get_crawler()
self.pipeline = ImagesPipeline(self.tempdir, crawler=crawler)
crawler = get_crawler(None, {"IMAGES_STORE": self.tempdir})
self.pipeline = build_from_crawler(ImagesPipeline, crawler)
def teardown_method(self):
rmtree(self.tempdir)
@ -137,8 +138,8 @@ class TestImagesPipeline:
) -> str:
return f"thumb/{thumb_id}/{item.get('path')}"
thumb_path = CustomImagesPipeline.from_crawler(
get_crawler(None, {"IMAGES_STORE": self.tempdir})
thumb_path = build_from_crawler(
CustomImagesPipeline, get_crawler(None, {"IMAGES_STORE": self.tempdir})
).thumb_path
item = {"path": "path-to-store-file"}
request = Request("http://example.com")
@ -316,8 +317,8 @@ class TestImagesPipeline:
],
)
def test_rejects_non_list_image_urls(self, tmp_path, bad_type):
pipeline = ImagesPipeline.from_crawler(
get_crawler(None, {"IMAGES_STORE": str(tmp_path)})
pipeline = build_from_crawler(
ImagesPipeline, get_crawler(None, {"IMAGES_STORE": str(tmp_path)})
)
item = ImagesPipelineTestItem()
item["image_urls"] = bad_type
@ -335,8 +336,8 @@ class TestImagesPipelineFieldsMixin(ABC):
def test_item_fields_default(self):
url = "http://www.example.com/images/1.jpg"
item = self.item_class(name="item1", image_urls=[url])
pipeline = ImagesPipeline.from_crawler(
get_crawler(None, {"IMAGES_STORE": "s3://example/images/"})
pipeline = build_from_crawler(
ImagesPipeline, get_crawler(None, {"IMAGES_STORE": "s3://example/images/"})
)
requests = list(pipeline.get_media_requests(item, DUMMY_SPIDER_INFO))
assert requests[0].url == url
@ -349,7 +350,8 @@ class TestImagesPipelineFieldsMixin(ABC):
def test_item_fields_override_settings(self):
url = "http://www.example.com/images/1.jpg"
item = self.item_class(name="item1", custom_image_urls=[url])
pipeline = ImagesPipeline.from_crawler(
pipeline = build_from_crawler(
ImagesPipeline,
get_crawler(
None,
{
@ -357,7 +359,7 @@ class TestImagesPipelineFieldsMixin(ABC):
"IMAGES_URLS_FIELD": "custom_image_urls",
"IMAGES_RESULT_FIELD": "custom_images",
},
)
),
)
requests = list(pipeline.get_media_requests(item, DUMMY_SPIDER_INFO))
assert requests[0].url == url
@ -493,8 +495,12 @@ class TestImagesPipelineCustomSettings:
have different settings.
"""
custom_settings = self._generate_fake_settings(tmp_path)
default_sts_pipe = ImagesPipeline(tmp_path, crawler=get_crawler(None))
user_sts_pipe = ImagesPipeline.from_crawler(get_crawler(None, custom_settings))
default_sts_pipe = build_from_crawler(
ImagesPipeline, get_crawler(None, {"IMAGES_STORE": tmp_path})
)
user_sts_pipe = build_from_crawler(
ImagesPipeline, get_crawler(None, custom_settings)
)
for pipe_attr, settings_attr in self.img_cls_attribute_names:
expected_default_value = self.default_pipeline_settings.get(pipe_attr)
custom_value = custom_settings.get(settings_attr)
@ -510,8 +516,8 @@ class TestImagesPipelineCustomSettings:
from class attributes.
"""
pipeline_cls = self._generate_fake_pipeline_subclass()
pipeline = pipeline_cls.from_crawler(
get_crawler(None, {"IMAGES_STORE": tmp_path})
pipeline = build_from_crawler(
pipeline_cls, get_crawler(None, {"IMAGES_STORE": tmp_path})
)
for pipe_attr, _ in self.img_cls_attribute_names:
# Instance attribute (lowercase) must be equal to class attribute (uppercase).
@ -526,7 +532,7 @@ class TestImagesPipelineCustomSettings:
"""
pipeline_cls = self._generate_fake_pipeline_subclass()
settings = self._generate_fake_settings(tmp_path)
pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
pipeline = build_from_crawler(pipeline_cls, get_crawler(None, settings))
for pipe_attr, settings_attr in self.img_cls_attribute_names:
# Instance attribute (lowercase) must be equal to
# value defined in settings.
@ -544,8 +550,8 @@ class TestImagesPipelineCustomSettings:
class UserDefinedImagePipeline(ImagesPipeline):
pass
user_pipeline = UserDefinedImagePipeline.from_crawler(
get_crawler(None, {"IMAGES_STORE": tmp_path})
user_pipeline = build_from_crawler(
UserDefinedImagePipeline, get_crawler(None, {"IMAGES_STORE": tmp_path})
)
for pipe_attr, _ in self.img_cls_attribute_names:
# Values from settings for custom pipeline should be set on pipeline instance.
@ -563,8 +569,8 @@ class TestImagesPipelineCustomSettings:
prefix = UserDefinedImagePipeline.__name__.upper()
settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = UserDefinedImagePipeline.from_crawler(
get_crawler(None, settings)
user_pipeline = build_from_crawler(
UserDefinedImagePipeline, get_crawler(None, settings)
)
for pipe_attr, settings_attr in self.img_cls_attribute_names:
# Values from settings for custom pipeline should be set on pipeline instance.
@ -580,7 +586,7 @@ class TestImagesPipelineCustomSettings:
pipeline_cls = self._generate_fake_pipeline_subclass()
prefix = pipeline_cls.__name__.upper()
settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
user_pipeline = build_from_crawler(pipeline_cls, get_crawler(None, settings))
for pipe_attr, settings_attr in self.img_cls_attribute_names:
custom_value = settings.get(prefix + "_" + settings_attr)
assert custom_value != self.default_pipeline_settings[pipe_attr]
@ -591,8 +597,8 @@ class TestImagesPipelineCustomSettings:
DEFAULT_IMAGES_URLS_FIELD = "something"
DEFAULT_IMAGES_RESULT_FIELD = "something_else"
pipeline = UserDefinedImagePipeline.from_crawler(
get_crawler(None, {"IMAGES_STORE": tmp_path})
pipeline = build_from_crawler(
UserDefinedImagePipeline, get_crawler(None, {"IMAGES_STORE": tmp_path})
)
assert (
pipeline.images_result_field
@ -613,7 +619,7 @@ class TestImagesPipelineCustomSettings:
class UserPipe(ImagesPipeline):
pass
pipeline_cls = UserPipe.from_crawler(get_crawler(None, settings))
pipeline_cls = build_from_crawler(UserPipe, get_crawler(None, settings))
for pipe_attr, settings_attr in self.img_cls_attribute_names:
expected_value = settings.get(settings_attr)
@ -632,7 +638,7 @@ class TestImagesPipelineCustomSettings:
},
)
ImagesPipeline.from_crawler(crawler)
build_from_crawler(ImagesPipeline, crawler)
assert S3FilesStore.POLICY == "public-read"
finally:
@ -651,7 +657,7 @@ class TestImagesPipelineCustomSettings:
},
)
ImagesPipeline.from_crawler(crawler)
build_from_crawler(ImagesPipeline, crawler)
assert GCSFilesStore.POLICY == "authenticatedRead"
finally:

11
tests/test_pipeline_media.py
View File

@ -10,8 +10,8 @@ from twisted.python.failure import Failure
from scrapy import signals
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.pipelines.files import FileException
from scrapy.pipelines.media import (
FileException,
FileInfo,
FileInfoOrError,
MediaPipeline,
@ -19,6 +19,7 @@ from scrapy.pipelines.media import (
)
from scrapy.utils.defer import _defer_sleep_async
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.signal import disconnect_all
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
@ -78,7 +79,7 @@ class TestBaseMediaPipeline:
crawler = get_crawler(DefaultSpider, self.settings)
crawler.spider = crawler._create_spider()
crawler.engine = MagicMock(download_async=mocked_download_func)
self.pipe = self.pipeline_class.from_crawler(crawler)
self.pipe = build_from_crawler(self.pipeline_class, crawler)
self.pipe.open_spider()
self.info = self.pipe.spiderinfo
assert crawler.request_fingerprinter is not None
@ -529,7 +530,7 @@ class TestBuildFromCrawler:
class Pipeline(UserDefinedPipeline):
pass
pipe = Pipeline.from_crawler(self.crawler)
pipe = build_from_crawler(Pipeline, self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
@ -549,7 +550,7 @@ class TestBuildFromCrawler:
o._from_crawler_called = True
return o
pipe = Pipeline.from_crawler(self.crawler)
pipe = build_from_crawler(Pipeline, self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert pipe._from_crawler_called
@ -568,7 +569,7 @@ class TestBuildFromCrawler:
o.store_uri = settings["FILES_STORE"]
return o
pipe = Pipeline.from_crawler(self.crawler)
pipe = build_from_crawler(Pipeline, self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert pipe._from_crawler_called

3
tests/test_pipelines.py
View File

@ -11,6 +11,7 @@ from scrapy.pipelines import ItemPipelineManager
from scrapy.utils.asyncio import call_later
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.mockserver.http import MockServer
@ -258,7 +259,7 @@ class TestCustomPipelineManager:
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
itemproc = CustomPipelineManager.from_crawler(crawler)
itemproc = build_from_crawler(CustomPipelineManager, crawler)
with pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager.process_item\(\) is deprecated, use process_item_async\(\)",

32
tests/test_pqueues.py
View File

@ -21,8 +21,8 @@ class TestPriorityQueue:
def test_queue_push_pop_one(self):
temp_dir = tempfile.mkdtemp()
queue = ScrapyPriorityQueue.from_crawler(
self.crawler, FifoMemoryQueue, temp_dir
queue = build_from_crawler(
ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir
)
assert queue.pop() is None
assert len(queue) == 0
@ -39,8 +39,8 @@ class TestPriorityQueue:
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
pytest.skip("queuelib.queue.FifoMemoryQueue.peek is defined")
temp_dir = tempfile.mkdtemp()
queue = ScrapyPriorityQueue.from_crawler(
self.crawler, FifoMemoryQueue, temp_dir
queue = build_from_crawler(
ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir
)
queue.push(Request("https://example.org"))
with pytest.raises(
@ -54,8 +54,8 @@ class TestPriorityQueue:
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
pytest.skip("queuelib.queue.FifoMemoryQueue.peek is undefined")
temp_dir = tempfile.mkdtemp()
queue = ScrapyPriorityQueue.from_crawler(
self.crawler, FifoMemoryQueue, temp_dir
queue = build_from_crawler(
ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir
)
assert len(queue) == 0
assert queue.peek() is None
@ -78,7 +78,8 @@ class TestPriorityQueue:
def test_init_prios_with_start_queue(self):
temp_dir = tempfile.mkdtemp()
queue = ScrapyPriorityQueue.from_crawler(
queue = build_from_crawler(
ScrapyPriorityQueue,
self.crawler,
PickleFifoDiskQueue,
temp_dir,
@ -88,7 +89,8 @@ class TestPriorityQueue:
queue.push(req)
startprios = queue.close()
queue2 = ScrapyPriorityQueue.from_crawler(
queue2 = build_from_crawler(
ScrapyPriorityQueue,
self.crawler,
PickleFifoDiskQueue,
temp_dir,
@ -101,8 +103,8 @@ class TestPriorityQueue:
def test_queue_push_pop_priorities(self):
temp_dir = tempfile.mkdtemp()
queue = ScrapyPriorityQueue.from_crawler(
self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3]
queue = build_from_crawler(
ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3]
)
assert queue.pop() is None
assert len(queue) == 0
@ -124,8 +126,9 @@ class TestDownloaderAwarePriorityQueue:
def setup_method(self):
crawler = get_crawler(Spider)
crawler.engine = Mock(downloader=MockDownloader())
self.queue = DownloaderAwarePriorityQueue.from_crawler(
crawler=crawler,
self.queue = build_from_crawler(
DownloaderAwarePriorityQueue,
crawler,
downstream_queue_cls=FifoMemoryQueue,
key="foo/bar",
)
@ -262,8 +265,9 @@ def test_slot_directory_removed_when_slot_drains(tmp_path):
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider("foo")
crawler.engine = Mock(downloader=MockDownloader())
queue = DownloaderAwarePriorityQueue.from_crawler(
crawler=crawler,
queue = build_from_crawler(
DownloaderAwarePriorityQueue,
crawler,
downstream_queue_cls=PickleFifoDiskQueue,
key=str(tmp_path),
)

5
tests/test_resolver.py
View File

@ -7,6 +7,7 @@ from twisted.internet.address import IPv4Address, IPv6Address
from scrapy.resolver import CachingHostnameResolver, CachingThreadedResolver, dnscache
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
@ -22,7 +23,7 @@ def reset_dnscache():
def test_caching_threaded_resolver_dnscache_disabled():
crawler = get_crawler(settings_dict={"DNSCACHE_ENABLED": False})
CachingThreadedResolver.from_crawler(crawler, Mock())
build_from_crawler(CachingThreadedResolver, crawler, Mock())
assert dnscache.limit == 0
@ -37,7 +38,7 @@ async def test_caching_threaded_resolver_getHostByName_cache_hit():
def test_caching_hostname_resolver_dnscache_disabled():
crawler = get_crawler(settings_dict={"DNSCACHE_ENABLED": False})
CachingHostnameResolver.from_crawler(crawler, Mock())
build_from_crawler(CachingHostnameResolver, crawler, Mock())
assert dnscache.limit == 0

13
tests/test_robotstxt_interface.py
View File

@ -1,6 +1,6 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, cast
import pytest
@ -12,6 +12,7 @@ from scrapy.robotstxt import (
decode_robotstxt,
)
from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
from scrapy.utils.misc import build_from_crawler
from tests.utils.robotstxt import rerp_available
if TYPE_CHECKING:
@ -20,6 +21,9 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
# The parser backends only use the crawler to get the spider to log with.
NO_CRAWLER = cast("Crawler", None)
class BaseRobotParserTest:
parser_cls: type[RobotParser]
@ -28,8 +32,7 @@ class BaseRobotParserTest:
self.parser_cls = parser_cls
def _parse(self, robotstxt_body: bytes) -> RobotParser:
# The parser backends only use the crawler to get the spider to log with.
return self.parser_cls.from_crawler(None, robotstxt_body) # type: ignore[arg-type]
return build_from_crawler(self.parser_cls, NO_CRAWLER, robotstxt_body)
def test_allowed(self):
robotstxt_robotstxt_body = (
@ -127,7 +130,9 @@ class TestRobotParser:
def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
return True
rp = AllowAllRobotParser()
rp = build_from_crawler(
AllowAllRobotParser, NO_CRAWLER, b"User-agent: *\nCrawl-delay: 10\n"
)
assert rp.crawl_delay("*") is None

10
tests/test_scheduler.py
View File

@ -15,7 +15,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.defer import ensure_awaitable
from scrapy.utils.misc import load_object
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import coroutine_test, inline_callbacks_test
@ -46,8 +46,8 @@ async def create_scheduler(
priority_queue_cls: str, jobdir: Path | None
) -> AsyncGenerator[Scheduler]:
mock_crawler = MockCrawler(priority_queue_cls, jobdir)
scheduler = Scheduler.from_crawler(mock_crawler)
spider = Spider(name="spider")
scheduler = build_from_crawler(Scheduler, mock_crawler)
spider = Spider.from_crawler(mock_crawler, name="spider")
await ensure_awaitable(scheduler.open(spider))
try:
yield scheduler
@ -333,8 +333,8 @@ class TestIncompatibility:
"CONCURRENT_REQUESTS_PER_IP": 1,
}
crawler = get_crawler(Spider, settings)
scheduler = Scheduler.from_crawler(crawler)
spider = Spider(name="spider")
scheduler = build_from_crawler(Scheduler, crawler)
spider = Spider.from_crawler(crawler, name="spider")
scheduler.open(spider)
def test_incompatibility(self):

11
tests/test_spidermiddleware.py
View File

@ -15,6 +15,7 @@ from scrapy.spiders import Spider
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
@ -31,7 +32,7 @@ class TestSpiderMiddleware:
self.response = Response(self.request.url, request=self.request)
self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES_BASE": {}})
self.crawler.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
self.mwman = build_from_crawler(SpiderMiddlewareManager, self.crawler)
async def _scrape_response(self) -> Any:
"""Execute spider mw manager's scrape_response_async method and return the result.
@ -139,7 +140,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting}
)
self.crawler.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
self.mwman = build_from_crawler(SpiderMiddlewareManager, self.crawler)
return await self.mwman.scrape_response_async(
self._scrape_func, self.response, self.request
)
@ -266,7 +267,7 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):
TestSpider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting}
)
self.crawler.spider = self.crawler._create_spider()
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
self.mwman = build_from_crawler(SpiderMiddlewareManager, self.crawler)
return await self.mwman.process_start()
@coroutine_test
@ -307,7 +308,7 @@ class TestUniversalMiddlewareManager:
@pytest.fixture
def mwman(self, crawler: Crawler) -> SpiderMiddlewareManager:
return SpiderMiddlewareManager.from_crawler(crawler)
return build_from_crawler(SpiderMiddlewareManager, crawler)
def test_simple_mw(self, mwman: SpiderMiddlewareManager) -> None:
mw = ProcessSpiderOutputSyncMiddleware()
@ -376,7 +377,7 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES": setting})
self.crawler.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
self.mwman = build_from_crawler(SpiderMiddlewareManager, self.crawler)
return await self.mwman.scrape_response_async(
self._scrape_func, self.response, self.request
)

9
tests/test_spidermiddleware_base.py
View File

@ -8,6 +8,7 @@ from scrapy import Request, Spider
from scrapy.http import Response
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
@ -25,7 +26,7 @@ async def test_trivial(crawler: Crawler) -> None:
class TrivialSpiderMiddleware(BaseSpiderMiddleware):
pass
mw = TrivialSpiderMiddleware.from_crawler(crawler)
mw = build_from_crawler(TrivialSpiderMiddleware, crawler)
assert hasattr(mw, "crawler")
assert mw.crawler is crawler
test_req = Request("data:,")
@ -49,7 +50,7 @@ async def test_processed_request(crawler: Crawler) -> None:
return Request("data:30,")
return request
mw = ProcessReqSpiderMiddleware.from_crawler(crawler)
mw = build_from_crawler(ProcessReqSpiderMiddleware, crawler)
test_req1 = Request("data:1,")
test_req2 = Request("data:2,")
test_req3 = Request("data:3,")
@ -81,7 +82,7 @@ async def test_processed_item(crawler: Crawler) -> None:
item["foo"] = 30
return item
mw = ProcessItemSpiderMiddleware.from_crawler(crawler)
mw = build_from_crawler(ProcessItemSpiderMiddleware, crawler)
test_req = Request("data:,")
spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]
for processed in [
@ -110,7 +111,7 @@ async def test_processed_both(crawler: Crawler) -> None:
item["foo"] = 30
return item
mw = ProcessBothSpiderMiddleware.from_crawler(crawler)
mw = build_from_crawler(ProcessBothSpiderMiddleware, crawler)
test_req1 = Request("data:1,")
test_req2 = Request("data:2,")
test_req3 = Request("data:3,")

2
tests/test_spidermiddleware_depth.py
View File

@ -36,7 +36,7 @@ def stats(crawler: Crawler) -> Generator[StatsCollector]:
@pytest.fixture
def mw(crawler: Crawler) -> DepthMiddleware:
return DepthMiddleware.from_crawler(crawler)
return build_from_crawler(DepthMiddleware, crawler)
def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> None:

9
tests/test_spidermiddleware_httperror.py
View File

@ -7,6 +7,7 @@ import pytest
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.spiders import MockServerSpider
@ -79,7 +80,7 @@ class TestHttpErrorMiddleware:
def mw(self) -> HttpErrorMiddleware:
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
return HttpErrorMiddleware.from_crawler(crawler)
return build_from_crawler(HttpErrorMiddleware, crawler)
def test_process_spider_input(
self, mw: HttpErrorMiddleware, res200: Response, res404: Response
@ -115,7 +116,7 @@ class TestHttpErrorMiddlewareSettings:
def mw(self) -> HttpErrorMiddleware:
crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOWED_CODES": (402,)})
crawler.spider = crawler._create_spider()
return HttpErrorMiddleware.from_crawler(crawler)
return build_from_crawler(HttpErrorMiddleware, crawler)
def test_process_spider_input(
self,
@ -155,7 +156,7 @@ class TestHttpErrorMiddlewareHandleAll:
def mw(self) -> HttpErrorMiddleware:
crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOW_ALL": True})
crawler.spider = crawler._create_spider()
return HttpErrorMiddleware.from_crawler(crawler)
return build_from_crawler(HttpErrorMiddleware, crawler)
def test_process_spider_input(
self,
@ -179,7 +180,7 @@ class TestHttpErrorMiddlewareHandleAll:
def test_httperror_allow_all_false(self) -> None:
crawler = get_crawler(_HttpErrorSpider)
mw = HttpErrorMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpErrorMiddleware, crawler)
request_httpstatus_false = Request(
"http://scrapytest.org", meta={"handle_httpstatus_all": False}
)

7
tests/test_spidermiddleware_metacopy.py
View File

@ -8,6 +8,7 @@ import pytest
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.metacopy import MetaCopyDetectionMiddleware
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@ -27,7 +28,7 @@ def crawler() -> Crawler:
@pytest.fixture
def mw(crawler: Crawler) -> MetaCopyDetectionMiddleware:
return MetaCopyDetectionMiddleware.from_crawler(crawler)
return build_from_crawler(MetaCopyDetectionMiddleware, crawler)
def process(
@ -94,7 +95,7 @@ class TestInternalKeysCheck:
def test_skip_keys_setting(self, caplog: pytest.LogCaptureFixture) -> None:
with caplog.at_level(WARNING):
crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})
mw = MetaCopyDetectionMiddleware.from_crawler(crawler)
mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)
req = Request("https://example.com/1", meta={"retry_times": 1})
process(mw, [req])
assert not caplog.records
@ -102,7 +103,7 @@ class TestInternalKeysCheck:
def test_skip_keys_setting_partial(self, caplog: pytest.LogCaptureFixture) -> None:
with caplog.at_level(WARNING):
crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})
mw = MetaCopyDetectionMiddleware.from_crawler(crawler)
mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)
req = Request(
"https://example.com/1",
meta={"retry_times": 1, "redirect_times": 2},

2
tests/test_spidermiddleware_urllength.py
View File

@ -37,7 +37,7 @@ def stats(crawler: Crawler) -> StatsCollector:
@pytest.fixture
def mw(crawler: Crawler) -> UrlLengthMiddleware:
return UrlLengthMiddleware.from_crawler(crawler)
return build_from_crawler(UrlLengthMiddleware, crawler)
def process_spider_output(mw: UrlLengthMiddleware) -> list[Request]:

3
tests/test_spiderstate.py
View File

@ -8,6 +8,7 @@ import pytest
from scrapy.exceptions import NotConfigured
from scrapy.extensions.spiderstate import SpiderState
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@ -49,4 +50,4 @@ def test_state_attribute() -> None:
def test_not_configured() -> None:
crawler = get_crawler(Spider)
with pytest.raises(NotConfigured):
SpiderState.from_crawler(crawler)
build_from_crawler(SpiderState, crawler)

21
tests/test_squeues_request.py
View File

@ -20,6 +20,7 @@ from scrapy.squeues import (
PickleFifoDiskQueue,
PickleLifoDiskQueue,
)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@ -114,8 +115,8 @@ class TestPickleFifoDiskQueueRequest(TestRequestQueueBase):
@pytest.fixture
def q(self, crawler, tmp_path):
queue = PickleFifoDiskQueue.from_crawler(
crawler=crawler, key=str(tmp_path / "pickle" / "fifo")
queue = build_from_crawler(
PickleFifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "fifo")
)
try:
yield queue
@ -128,8 +129,8 @@ class TestPickleLifoDiskQueueRequest(TestRequestQueueBase):
@pytest.fixture
def q(self, crawler, tmp_path):
queue = PickleLifoDiskQueue.from_crawler(
crawler=crawler, key=str(tmp_path / "pickle" / "lifo")
queue = build_from_crawler(
PickleLifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "lifo")
)
try:
yield queue
@ -142,8 +143,8 @@ class TestMarshalFifoDiskQueueRequest(TestRequestQueueBase):
@pytest.fixture
def q(self, crawler, tmp_path):
queue = MarshalFifoDiskQueue.from_crawler(
crawler=crawler, key=str(tmp_path / "marshal" / "fifo")
queue = build_from_crawler(
MarshalFifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "fifo")
)
try:
yield queue
@ -156,8 +157,8 @@ class TestMarshalLifoDiskQueueRequest(TestRequestQueueBase):
@pytest.fixture
def q(self, crawler, tmp_path):
queue = MarshalLifoDiskQueue.from_crawler(
crawler=crawler, key=str(tmp_path / "marshal" / "lifo")
queue = build_from_crawler(
MarshalLifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "lifo")
)
try:
yield queue
@ -170,7 +171,7 @@ class TestFifoMemoryQueueRequest(TestRequestQueueBase):
@pytest.fixture
def q(self, crawler):
return FifoMemoryQueue.from_crawler(crawler=crawler)
return build_from_crawler(FifoMemoryQueue, crawler)
class TestLifoMemoryQueueRequest(TestRequestQueueBase):
@ -178,4 +179,4 @@ class TestLifoMemoryQueueRequest(TestRequestQueueBase):
@pytest.fixture
def q(self, crawler):
return LifoMemoryQueue.from_crawler(crawler=crawler)
return build_from_crawler(LifoMemoryQueue, crawler)

5
tests/test_stats.py
View File

@ -10,6 +10,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.corestats import CoreStats
from scrapy.spiders import Spider
from scrapy.statscollectors import DummyStatsCollector, StatsCollector
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
from tests.utils.decorators import coroutine_test
@ -41,7 +42,7 @@ class TestCoreStatsExtension:
fixed_datetime = datetime(2019, 12, 1, 11, 38)
mock_datetime.now = mock.Mock(return_value=fixed_datetime)
crawler.stats = StatsCollector(crawler)
ext = CoreStats.from_crawler(crawler)
ext = build_from_crawler(CoreStats, crawler)
ext.spider_opened(spider)
ext.item_scraped({}, spider)
ext.response_received(spider)
@ -62,7 +63,7 @@ class TestCoreStatsExtension:
self, crawler: Crawler, spider: Spider
) -> None:
crawler.stats = DummyStatsCollector(crawler)
ext = CoreStats.from_crawler(crawler)
ext = build_from_crawler(CoreStats, crawler)
ext.spider_opened(spider)
ext.item_scraped({}, spider)
ext.response_received(spider)

20
tests/test_utils_log.py
View File

@ -4,12 +4,14 @@ import json
import logging
import re
import sys
import warnings
from io import StringIO
from typing import TYPE_CHECKING, Any, cast
import pytest
from twisted.python.failure import Failure
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.log import (
LogCounterHandler,
SpiderLoggerAdapter,
@ -332,7 +334,9 @@ class TestLogformatterAdapter:
"LogFormatterResult",
{"level": logging.INFO, "msg": "90% done", "args": args},
)
assert self._log(caplog, logkws) == "90% done"
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
assert self._log(caplog, logkws) == "90% done"
@pytest.mark.parametrize(
("msg", "args"),
@ -345,4 +349,16 @@ class TestLogformatterAdapter:
args: dict[str, Any] | tuple[Any, ...],
) -> None:
logkws: LogFormatterResult = {"level": logging.INFO, "msg": msg, "args": args}
assert self._log(caplog, logkws) == "90% done"
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
assert self._log(caplog, logkws) == "90% done"
def test_msg_mapping_placeholders_without_args(
self, caplog: pytest.LogCaptureFixture
) -> None:
logkws = cast(
"LogFormatterResult",
{"level": logging.INFO, "msg": "%(pct)d%% done", "pct": 90},
)
with pytest.warns(ScrapyDeprecationWarning, match="no args"):
assert self._log(caplog, logkws) == "90% done"

10
tests/test_utils_serialize.py
View File

@ -20,11 +20,17 @@ class TestJsonEncoder:
def test_encode_decode(self, encoder: ScrapyJSONEncoder) -> None:
dt = datetime.datetime(2010, 1, 2, 10, 11, 12)
dts = "2010-01-02 10:11:12"
dts = "2010-01-02T10:11:12"
dt_aware = datetime.datetime(
2010, 1, 2, 10, 11, 12, 133700, tzinfo=datetime.timezone.utc
)
dt_awares = "2010-01-02T10:11:12.133700+00:00"
d = datetime.date(2010, 1, 2)
ds = "2010-01-02"
t = datetime.time(10, 11, 12)
ts = "10:11:12"
t_us = datetime.time(10, 11, 12, 133700)
t_uss = "10:11:12.133700"
dec = Decimal("1000.12")
decs = "1000.12"
s = {"foo"}
@ -36,7 +42,9 @@ class TestJsonEncoder:
("foo", "foo"),
(d, ds),
(t, ts),
(t_us, t_uss),
(dt, dts),
(dt_aware, dt_awares),
(dec, decs),
(["foo", d], ["foo", ds]),
(s, ss),

39
tests/utils/bases/download_handlers_http.py
View File

@ -11,7 +11,7 @@ from contextlib import asynccontextmanager
from http import HTTPStatus
from ipaddress import IPv4Address
from socket import gethostbyname
from typing import TYPE_CHECKING, Any, ClassVar
from typing import TYPE_CHECKING, Any, ClassVar, Literal
from urllib.parse import urlparse
import pytest
@ -60,6 +60,9 @@ if TYPE_CHECKING:
from tests.mockserver.http import MockServer
BadHeaderHandling = Literal["skip-bad", "skip-rest", "fail"]
class TestHttpBase(ABC):
is_secure: bool = False
http2: bool = False
@ -72,6 +75,14 @@ class TestHttpBase(ABC):
# h2.connection.H2Connection.receive_data()), thus closing all streams that
# were using it, and we handle this as a normal exception.
handler_supports_http2_dataloss: bool = True
# What the handler does with a bad response header line, e.g. one with no
# colon in it:
# "skip-bad": the bad line is skipped and the header lines that follow it
# are still parsed, which is what web browsers do;
# "skip-rest": the bad line is skipped along with the header lines that
# follow it;
# "fail": the response cannot be downloaded at all.
handler_bad_header_handling: BadHeaderHandling = "skip-bad"
# default headers added by the underlying library that cannot be suppressed
always_present_req_headers: ClassVar[frozenset[str]] = frozenset()
default_handler_settings: ClassVar[dict[str, Any]] = {}
@ -645,6 +656,32 @@ class TestHttpBase(ABC):
in caplog.text
)
@coroutine_test
async def test_download_bad_header(self, mockserver: MockServer) -> None:
if self.http2:
pytest.skip("Header lines are specific to HTTP/1.x")
request = Request(mockserver.url("/bad-header", is_secure=self.is_secure))
async with self.get_dh() as download_handler:
if self.handler_bad_header_handling == "fail":
with pytest.raises(DownloadFailedError):
await download_handler.download_request(request)
return
response = await download_handler.download_request(request)
assert response.status == 200
assert response.body == b"Works"
# the header line that precedes the bad one
assert response.headers.get(b"Content-Type") == b"text/html"
# the header split into two lines, also before the bad one
folded_header = response.headers.get(b"X-Folded-Header")
assert folded_header is not None
# the separator between both parts depends on the handler
assert folded_header.split() == [b"one", b"two"]
# the header line that follows the bad one
expected_value = (
b"works" if self.handler_bad_header_handling == "skip-bad" else None
)
assert response.headers.get(b"X-After-Bad-Header") == expected_value
@coroutine_test
async def test_download_chunked_content(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/chunked", is_secure=self.is_secure))

74
tests/utils/bases/redirect.py
View File

@ -8,7 +8,7 @@ import pytest
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.http import Request, Response
from scrapy.utils.misc import set_environ
from scrapy.utils.misc import build_from_crawler, set_environ
from scrapy.utils.test import get_crawler
@ -221,8 +221,8 @@ class TestRedirectBase(ABC):
def test_meta_proxy_http_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("http://example.com", meta=meta)
@ -262,8 +262,8 @@ class TestRedirectBase(ABC):
def test_meta_proxy_http_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("http://example.com", meta=meta)
@ -303,8 +303,8 @@ class TestRedirectBase(ABC):
def test_meta_proxy_https_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("https://example.com", meta=meta)
@ -344,8 +344,8 @@ class TestRedirectBase(ABC):
def test_meta_proxy_https_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("https://example.com", meta=meta)
@ -385,8 +385,8 @@ class TestRedirectBase(ABC):
def test_meta_proxy_http_to_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("http://example.com", meta=meta)
@ -426,8 +426,8 @@ class TestRedirectBase(ABC):
def test_meta_proxy_https_to_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("https://example.com", meta=meta)
@ -467,12 +467,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_http_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
@ -511,12 +511,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_http_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
@ -555,12 +555,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_https_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"https_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
@ -599,12 +599,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_https_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"https_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
@ -643,13 +643,13 @@ class TestRedirectBase(ABC):
def test_system_proxy_proxied_http_to_proxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"http_proxy": "https://a:@a.example",
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
@ -688,12 +688,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_proxied_http_to_unproxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
@ -732,12 +732,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_unproxied_http_to_proxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
@ -776,8 +776,8 @@ class TestRedirectBase(ABC):
def test_system_proxy_unproxied_http_to_unproxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
@ -816,13 +816,13 @@ class TestRedirectBase(ABC):
def test_system_proxy_proxied_https_to_proxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"http_proxy": "https://a:@a.example",
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
@ -861,12 +861,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_proxied_https_to_unproxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
@ -905,12 +905,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_unproxied_https_to_proxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
@ -949,8 +949,8 @@ class TestRedirectBase(ABC):
def test_system_proxy_unproxied_https_to_unproxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)

2
tox.ini
View File

@ -5,7 +5,7 @@
[tox]
requires =
sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10
sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.11
tox-uv
envlist =
pre-commit