diff --git a/scrapy/addons.py b/scrapy/addons.py index 2e12f8c8a..470be47c1 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -64,7 +64,7 @@ class AddonManager: :param settings: The :class:`~scrapy.settings.BaseSettings` object from \ which to read the early add-on configuration - :type settings: :class:`~scrapy.settings.Settings` + :type settings: :class:`~scrapy.settings.BaseSettings` """ for clspath in build_component_list(settings["ADDONS"]): addoncls = load_object(clspath) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 9064aa53a..07ff4a74e 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -2,9 +2,9 @@ An asynchronous FTP file download handler for scrapy which somehow emulates an http response. FTP connection parameters are passed using the request meta field: -- ftp_user (required) -- ftp_password (required) -- ftp_passive (by default, enabled) sets FTP connection passive mode +- ftp_user (optional, falls back to FTP_USER) +- ftp_password (optional, falls back to FTP_PASSWORD) +- ftp_passive (optional, falls back to FTP_PASSIVE_MODE) sets FTP connection passive mode - ftp_local_filename - If not given, file data will come in the response.body, as a normal scrapy Response, which will imply that the entire file will be on memory. diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 2b1721ced..6ca04a50e 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -60,7 +60,7 @@ else: class HttpCompressionMiddleware: - """This middleware allows compressed (gzip, deflate) traffic to be + """This middleware allows compressed (gzip, deflate etc.) traffic to be sent/received from websites""" def __init__( diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index ac5651dc7..a0a0b60a2 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -67,8 +67,9 @@ def get_retry_request( and :ref:`stats `, and to provide extra logging context (see :func:`logging.debug`). - *reason* is a string or an :class:`Exception` object that indicates the - reason why the request needs to be retried. It is used to name retry stats. + *reason* is a string, an :class:`Exception` subclass or an + :class:`Exception` object that indicates the reason why the request needs + to be retried. It is used to name retry stats. *max_retry_times* is a number that determines the maximum number of times that *request* can be retried. If not specified or ``None``, the number is diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index cdb0671ae..542ff1cdc 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -116,7 +116,7 @@ class AutoThrottle: # It works better with problematic sites. new_delay = max(target_delay, new_delay) - # Make sure self.mindelay <= new_delay <= self.max_delay + # Make sure self.mindelay <= new_delay <= self.maxdelay new_delay = min(max(self.mindelay, new_delay), self.maxdelay) # Dont adjust delay if response status != 200 and new delay is smaller diff --git a/scrapy/shell.py b/scrapy/shell.py index f7ec8bea0..dfea00c46 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -75,7 +75,7 @@ if TYPE_CHECKING: # running event loop. # # Side note: it should be possible to remove _request_deferred() by using -# engine.download_async() instead of engine.schedule(), losing the usual stuff +# engine.download_async() instead of engine.crawl(), losing the usual stuff # like spider middlewares (none of which should be important). # # Other architecture problems: diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 94b6dfbb5..156b73e7e 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) class HttpError(IgnoreRequest): - """A non-200 response was filtered""" + """A non-2xx response was filtered""" def __init__(self, response: Response, *args: Any, **kwargs: Any): self.response = response diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 935ac8bc1..d0259b634 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -523,7 +523,7 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: async def parse(self, response): deferred = some_dfd_helper() - result = await deferred_to_future(deferred) + result = await maybe_deferred_to_future(deferred) """ if not is_asyncio_available(): return d