mirror of https://github.com/scrapy/scrapy.git
get_retry_request: add typing information
This commit is contained in:
parent
dfd4ab2b6c
commit
c0f3ca1938
|
|
@ -9,8 +9,8 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
|
|||
Failed pages are collected on the scraping process and rescheduled at the end,
|
||||
once the spider has finished crawling all regular (non failed) pages.
|
||||
"""
|
||||
import logging
|
||||
from inspect import isclass
|
||||
from logging import getLogger, Logger
|
||||
from typing import Optional, Union
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.error import (
|
||||
|
|
@ -24,24 +24,26 @@ from twisted.internet.error import (
|
|||
)
|
||||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.response import response_status_message
|
||||
from scrapy.core.downloader.handlers.http11 import TunnelError
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.python import global_object_name
|
||||
from scrapy.utils.response import response_status_message
|
||||
|
||||
|
||||
retry_logger = logging.getLogger(__name__)
|
||||
retry_logger = getLogger(__name__)
|
||||
|
||||
|
||||
def get_retry_request(
|
||||
request,
|
||||
request: Request,
|
||||
*,
|
||||
spider,
|
||||
reason='unspecified',
|
||||
max_retry_times=None,
|
||||
priority_adjust=None,
|
||||
logger=retry_logger,
|
||||
stats_base_key='retry',
|
||||
spider: Spider,
|
||||
reason: Union[str, Exception] = 'unspecified',
|
||||
max_retry_times: Optional[int] = None,
|
||||
priority_adjust: Union[int, float, None] = None,
|
||||
logger: Logger = retry_logger,
|
||||
stats_base_key: str = 'retry',
|
||||
):
|
||||
"""
|
||||
Returns a new :class:`~scrapy.Request` object to retry the specified
|
||||
|
|
@ -105,7 +107,7 @@ def get_retry_request(
|
|||
priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
|
||||
new_request.priority = request.priority + priority_adjust
|
||||
|
||||
if isclass(reason):
|
||||
if callable(reason):
|
||||
reason = reason()
|
||||
if isinstance(reason, Exception):
|
||||
reason = global_object_name(reason.__class__)
|
||||
|
|
|
|||
Loading…
Reference in New Issue