diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 55dbcabec..ee51300cc 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -2,29 +2,38 @@ import ipaddress import itertools import logging from collections import deque -from typing import Dict, Optional +from typing import Dict, List, Optional from h2.config import H2Configuration from h2.connection import H2Connection from h2.events import ( - DataReceived, ResponseReceived, SettingsAcknowledged, + Event, DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) from h2.exceptions import ProtocolError from twisted.internet.defer import Deferred -from twisted.internet.protocol import connectionDone, Protocol +from twisted.internet.protocol import connectionDone, Factory, Protocol from twisted.internet.ssl import Certificate from twisted.python.failure import Failure +from twisted.web.client import URI from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request +from scrapy.settings import Settings logger = logging.getLogger(__name__) class H2ClientProtocol(Protocol): - def __init__(self) -> None: + def __init__(self, uri: URI, settings: Settings) -> None: + """ + Arguments: + uri -- URI of the base url to which HTTP/2 Connection will be made. + uri is used to verify that incoming client requests have correct + base URL. + settings -- Scrapy project settings + """ config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) @@ -53,8 +62,9 @@ class H2ClientProtocol(Protocol): self.metadata: H2ConnectionMetadataDict = { 'certificate': None, 'ip_address': None, - 'hostname': None, - 'port': None, + 'uri': uri, + 'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'), + 'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'), } @property @@ -135,8 +145,6 @@ class H2ClientProtocol(Protocol): destination = self.transport.getPeer() logger.debug('Connection made to {}'.format(destination)) self.metadata['ip_address'] = ipaddress.ip_address(destination.host) - self.metadata['port'] = destination.port - self.metadata['hostname'] = self.transport.transport.addr[0] self.conn.initiate_connection() self._write_to_transport() @@ -179,7 +187,7 @@ class H2ClientProtocol(Protocol): self._pending_request_stream_pool.clear() self.conn.close_connection() - def _handle_events(self, events: list) -> None: + def _handle_events(self, events: List[Event]) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler @@ -232,3 +240,12 @@ class H2ClientProtocol(Protocol): # Send leftover data for all the streams for stream in self.streams.values(): stream.receive_window_update() + + +class H2ClientFactory(Factory): + def __init__(self, uri: URI, settings: Settings) -> None: + self.uri = uri + self.settings = settings + + def buildProtocol(self, addr) -> H2ClientProtocol: + return H2ClientProtocol(self.uri, self.settings) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 8b66d4b85..5017f9cd4 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -16,6 +16,7 @@ from scrapy.core.http2.types import H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes +from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -34,7 +35,7 @@ class InactiveStreamClosed(ConnectionClosed): class InvalidHostname(Exception): - def __init__(self, request: Request, expected_hostname: Optional[str], expected_netloc: Optional[str]) -> None: + def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None: self.request = request self.expected_hostname = expected_hostname self.expected_netloc = expected_netloc @@ -83,10 +84,7 @@ class Stream: self, stream_id: int, request: Request, - protocol: "H2ClientProtocol", - download_maxsize: int = 0, - download_warnsize: int = 0, - fail_on_data_loss: bool = True + protocol: "H2ClientProtocol" ) -> None: """ Arguments: @@ -98,9 +96,14 @@ class Stream: self._request: Request = request self._protocol: "H2ClientProtocol" = protocol - self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) - self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) - self._fail_on_dataloss = self._request.meta.get('download_fail_on_dataloss', fail_on_data_loss) + self._download_maxsize = self._request.meta.get( + 'download_maxsize', + self._protocol.metadata['default_download_maxsize'] + ) + self._download_warnsize = self._request.meta.get( + 'download_warnsize', + self._protocol.metadata['default_download_warnsize'] + ) self.request_start_time = None @@ -174,9 +177,9 @@ class Stream: # Make sure that we are sending the request to the correct URL url = urlparse(self._request.url) return ( - url.netloc == self._protocol.metadata['hostname'] - or url.netloc == f'{self._protocol.metadata["hostname"]}:{self._protocol.metadata["port"]}' - or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' + url.netloc == to_unicode(self._protocol.metadata['uri'].host) + or url.netloc == to_unicode(self._protocol.metadata['uri'].netloc) + or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' ) def _get_request_headers(self) -> List[Tuple[str, str]]: @@ -391,8 +394,8 @@ class Stream: elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( self._request, - self._protocol.metadata['hostname'], - f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' + to_unicode(self._protocol.metadata['uri'].host), + f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' )) def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None: diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py index dd7b1187b..d2aa1a9d8 100644 --- a/scrapy/core/http2/types.py +++ b/scrapy/core/http2/types.py @@ -3,6 +3,7 @@ from ipaddress import IPv4Address, IPv6Address from typing import Union, Optional from twisted.internet.ssl import Certificate +from twisted.web.client import URI # for python < 3.8 -- typing.TypedDict is undefined from typing_extensions import TypedDict @@ -19,14 +20,16 @@ class H2ConnectionMetadataDict(TypedDict): # is updated when HTTP/2 connection is made successfully ip_address: Optional[Union[IPv4Address, IPv6Address]] - # Name of the peer HTTP/2 connection is established - hostname: Optional[str] + # URI of the peer HTTP/2 connection is made + uri: URI - port: Optional[int] - - # Both ip_address and hostname are used by the Stream before + # Both ip_address and uri are used by the Stream before # initiating the request to verify that the base address + # Variables taken from Project Settings + default_download_maxsize: int + default_download_warnsize: int + class H2ResponseDict(TypedDict): # Data received frame by frame from the server is appended diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 9efca5267..05e5f5047 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -11,17 +11,18 @@ from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint -from twisted.internet.protocol import Factory from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase +from twisted.web.client import URI from twisted.web.http import Request as TxRequest from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File -from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.core.http2.protocol import H2ClientFactory from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest +from scrapy.settings import Settings from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -183,7 +184,8 @@ class Https2ClientProtocolTestCase(TestCase): trustRoot=self.client_certificate, acceptableProtocols=[b'h2'] ) - h2_client_factory = Factory.forProtocol(H2ClientProtocol) + uri = URI.fromBytes(to_bytes(self.get_url('/'))) + h2_client_factory = H2ClientFactory(uri, Settings()) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory)