This commit is contained in:
Andrey Rakhmatullin 2026-06-30 14:47:55 +05:00
parent 23e67837e3
commit d4a404308d
8 changed files with 12 additions and 11 deletions

View File

@ -64,7 +64,7 @@ class AddonManager:
:param settings: The :class:`~scrapy.settings.BaseSettings` object from \
which to read the early add-on configuration
:type settings: :class:`~scrapy.settings.Settings`
:type settings: :class:`~scrapy.settings.BaseSettings`
"""
for clspath in build_component_list(settings["ADDONS"]):
addoncls = load_object(clspath)

View File

@ -2,9 +2,9 @@
An asynchronous FTP file download handler for scrapy which somehow emulates an http response.
FTP connection parameters are passed using the request meta field:
- ftp_user (required)
- ftp_password (required)
- ftp_passive (by default, enabled) sets FTP connection passive mode
- ftp_user (optional, falls back to FTP_USER)
- ftp_password (optional, falls back to FTP_PASSWORD)
- ftp_passive (optional, falls back to FTP_PASSIVE_MODE) sets FTP connection passive mode
- ftp_local_filename
- If not given, file data will come in the response.body, as a normal scrapy Response,
which will imply that the entire file will be on memory.

View File

@ -60,7 +60,7 @@ else:
class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
"""This middleware allows compressed (gzip, deflate etc.) traffic to be
sent/received from websites"""
def __init__(

View File

@ -67,8 +67,9 @@ def get_retry_request(
and :ref:`stats <topics-stats>`, and to provide extra logging context (see
:func:`logging.debug`).
*reason* is a string or an :class:`Exception` object that indicates the
reason why the request needs to be retried. It is used to name retry stats.
*reason* is a string, an :class:`Exception` subclass or an
:class:`Exception` object that indicates the reason why the request needs
to be retried. It is used to name retry stats.
*max_retry_times* is a number that determines the maximum number of times
that *request* can be retried. If not specified or ``None``, the number is

View File

@ -116,7 +116,7 @@ class AutoThrottle:
# It works better with problematic sites.
new_delay = max(target_delay, new_delay)
# Make sure self.mindelay <= new_delay <= self.max_delay
# Make sure self.mindelay <= new_delay <= self.maxdelay
new_delay = min(max(self.mindelay, new_delay), self.maxdelay)
# Dont adjust delay if response status != 200 and new delay is smaller

View File

@ -75,7 +75,7 @@ if TYPE_CHECKING:
# running event loop.
#
# Side note: it should be possible to remove _request_deferred() by using
# engine.download_async() instead of engine.schedule(), losing the usual stuff
# engine.download_async() instead of engine.crawl(), losing the usual stuff
# like spider middlewares (none of which should be important).
#
# Other architecture problems:

View File

@ -28,7 +28,7 @@ logger = logging.getLogger(__name__)
class HttpError(IgnoreRequest):
"""A non-200 response was filtered"""
"""A non-2xx response was filtered"""
def __init__(self, response: Response, *args: Any, **kwargs: Any):
self.response = response

View File

@ -523,7 +523,7 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]:
async def parse(self, response):
deferred = some_dfd_helper()
result = await deferred_to_future(deferred)
result = await maybe_deferred_to_future(deferred)
"""
if not is_asyncio_available():
return d