Full typing for scrapy/core/downloader/handlers/http10.py.

This commit is contained in:
Andrey Rakhmatullin 2024-05-04 18:09:25 +05:00
parent e8e13ebb78
commit af3e38ab1f
1 changed files with 18 additions and 9 deletions

View File

@ -3,8 +3,13 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Type
from twisted.internet.defer import Deferred
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import to_unicode
@ -12,29 +17,33 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
class HTTP10DownloadHandler:
lazy = False
def __init__(self, settings, crawler):
self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"])
self.ClientContextFactory = load_object(
def __init__(self, settings: BaseSettings, crawler: Crawler):
self.HTTPClientFactory: Type[ScrapyHTTPClientFactory] = load_object(
settings["DOWNLOADER_HTTPCLIENTFACTORY"]
)
self.ClientContextFactory: Type[ScrapyClientContextFactory] = load_object(
settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]
)
self._settings = settings
self._crawler = crawler
self._settings: BaseSettings = settings
self._crawler: Crawler = crawler
@classmethod
def from_crawler(cls, crawler) -> Self:
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings, crawler)
def download_request(self, request, spider):
def download_request(self, request: Request, spider: Spider) -> Deferred:
"""Return a deferred for the HTTP download"""
factory = self.HTTPClientFactory(request)
self._connect(factory)
return factory.deferred
def _connect(self, factory):
def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred:
from twisted.internet import reactor
host, port = to_unicode(factory.host), factory.port