mirror of https://github.com/scrapy/scrapy.git
Round 2.
This commit is contained in:
parent
23e67837e3
commit
d4a404308d
|
|
@ -64,7 +64,7 @@ class AddonManager:
|
|||
|
||||
:param settings: The :class:`~scrapy.settings.BaseSettings` object from \
|
||||
which to read the early add-on configuration
|
||||
:type settings: :class:`~scrapy.settings.Settings`
|
||||
:type settings: :class:`~scrapy.settings.BaseSettings`
|
||||
"""
|
||||
for clspath in build_component_list(settings["ADDONS"]):
|
||||
addoncls = load_object(clspath)
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@
|
|||
An asynchronous FTP file download handler for scrapy which somehow emulates an http response.
|
||||
|
||||
FTP connection parameters are passed using the request meta field:
|
||||
- ftp_user (required)
|
||||
- ftp_password (required)
|
||||
- ftp_passive (by default, enabled) sets FTP connection passive mode
|
||||
- ftp_user (optional, falls back to FTP_USER)
|
||||
- ftp_password (optional, falls back to FTP_PASSWORD)
|
||||
- ftp_passive (optional, falls back to FTP_PASSIVE_MODE) sets FTP connection passive mode
|
||||
- ftp_local_filename
|
||||
- If not given, file data will come in the response.body, as a normal scrapy Response,
|
||||
which will imply that the entire file will be on memory.
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ else:
|
|||
|
||||
|
||||
class HttpCompressionMiddleware:
|
||||
"""This middleware allows compressed (gzip, deflate) traffic to be
|
||||
"""This middleware allows compressed (gzip, deflate etc.) traffic to be
|
||||
sent/received from websites"""
|
||||
|
||||
def __init__(
|
||||
|
|
|
|||
|
|
@ -67,8 +67,9 @@ def get_retry_request(
|
|||
and :ref:`stats <topics-stats>`, and to provide extra logging context (see
|
||||
:func:`logging.debug`).
|
||||
|
||||
*reason* is a string or an :class:`Exception` object that indicates the
|
||||
reason why the request needs to be retried. It is used to name retry stats.
|
||||
*reason* is a string, an :class:`Exception` subclass or an
|
||||
:class:`Exception` object that indicates the reason why the request needs
|
||||
to be retried. It is used to name retry stats.
|
||||
|
||||
*max_retry_times* is a number that determines the maximum number of times
|
||||
that *request* can be retried. If not specified or ``None``, the number is
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ class AutoThrottle:
|
|||
# It works better with problematic sites.
|
||||
new_delay = max(target_delay, new_delay)
|
||||
|
||||
# Make sure self.mindelay <= new_delay <= self.max_delay
|
||||
# Make sure self.mindelay <= new_delay <= self.maxdelay
|
||||
new_delay = min(max(self.mindelay, new_delay), self.maxdelay)
|
||||
|
||||
# Dont adjust delay if response status != 200 and new delay is smaller
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ if TYPE_CHECKING:
|
|||
# running event loop.
|
||||
#
|
||||
# Side note: it should be possible to remove _request_deferred() by using
|
||||
# engine.download_async() instead of engine.schedule(), losing the usual stuff
|
||||
# engine.download_async() instead of engine.crawl(), losing the usual stuff
|
||||
# like spider middlewares (none of which should be important).
|
||||
#
|
||||
# Other architecture problems:
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
|
||||
class HttpError(IgnoreRequest):
|
||||
"""A non-200 response was filtered"""
|
||||
"""A non-2xx response was filtered"""
|
||||
|
||||
def __init__(self, response: Response, *args: Any, **kwargs: Any):
|
||||
self.response = response
|
||||
|
|
|
|||
|
|
@ -523,7 +523,7 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]:
|
|||
|
||||
async def parse(self, response):
|
||||
deferred = some_dfd_helper()
|
||||
result = await deferred_to_future(deferred)
|
||||
result = await maybe_deferred_to_future(deferred)
|
||||
"""
|
||||
if not is_asyncio_available():
|
||||
return d
|
||||
|
|
|
|||
Loading…
Reference in New Issue