feat: H2ClientFactory

scrapy/core/http2/protocol.py

@@ -2,29 +2,38 @@ import ipaddress
 import itertools
 import logging
 from collections import deque
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
 from h2.events import (
-    DataReceived, ResponseReceived, SettingsAcknowledged,
+    Event, DataReceived, ResponseReceived, SettingsAcknowledged,
     StreamEnded, StreamReset, WindowUpdated
 )
 from h2.exceptions import ProtocolError
 from twisted.internet.defer import Deferred
-from twisted.internet.protocol import connectionDone, Protocol
+from twisted.internet.protocol import connectionDone, Factory, Protocol
 from twisted.internet.ssl import Certificate
 from twisted.python.failure import Failure
+from twisted.web.client import URI
 
 from scrapy.core.http2.stream import Stream, StreamCloseReason
 from scrapy.core.http2.types import H2ConnectionMetadataDict
 from scrapy.http import Request
+from scrapy.settings import Settings
 
 logger = logging.getLogger(__name__)
 
 
 class H2ClientProtocol(Protocol):
-    def __init__(self) -> None:
+    def __init__(self, uri: URI, settings: Settings) -> None:
+        """
+        Arguments:
+            uri -- URI of the base url to which HTTP/2 Connection will be made.
+                uri is used to verify that incoming client requests have correct
+                base URL.
+            settings -- Scrapy project settings
+        """
         config = H2Configuration(client_side=True, header_encoding='utf-8')
         self.conn = H2Connection(config=config)
 
@@ -53,8 +62,9 @@ class H2ClientProtocol(Protocol):
         self.metadata: H2ConnectionMetadataDict = {
             'certificate': None,
             'ip_address': None,
-            'hostname': None,
-            'port': None,
+            'uri': uri,
+            'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'),
+            'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'),
         }
 
     @property
@@ -135,8 +145,6 @@ class H2ClientProtocol(Protocol):
         destination = self.transport.getPeer()
         logger.debug('Connection made to {}'.format(destination))
         self.metadata['ip_address'] = ipaddress.ip_address(destination.host)
-        self.metadata['port'] = destination.port
-        self.metadata['hostname'] = self.transport.transport.addr[0]
 
         self.conn.initiate_connection()
         self._write_to_transport()
@@ -179,7 +187,7 @@ class H2ClientProtocol(Protocol):
         self._pending_request_stream_pool.clear()
         self.conn.close_connection()
 
-    def _handle_events(self, events: list) -> None:
+    def _handle_events(self, events: List[Event]) -> None:
         """Private method which acts as a bridge between the events
         received from the HTTP/2 data and IH2EventsHandler
 
@@ -232,3 +240,12 @@ class H2ClientProtocol(Protocol):
             # Send leftover data for all the streams
             for stream in self.streams.values():
                 stream.receive_window_update()
+
+
+class H2ClientFactory(Factory):
+    def __init__(self, uri: URI, settings: Settings) -> None:
+        self.uri = uri
+        self.settings = settings
+
+    def buildProtocol(self, addr) -> H2ClientProtocol:
+        return H2ClientProtocol(self.uri, self.settings)

scrapy/core/http2/stream.py

@@ -16,6 +16,7 @@ from scrapy.core.http2.types import H2ResponseDict
 from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
+from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
     from scrapy.core.http2.protocol import H2ClientProtocol
@@ -34,7 +35,7 @@ class InactiveStreamClosed(ConnectionClosed):
 
 class InvalidHostname(Exception):
 
-    def __init__(self, request: Request, expected_hostname: Optional[str], expected_netloc: Optional[str]) -> None:
+    def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None:
         self.request = request
         self.expected_hostname = expected_hostname
         self.expected_netloc = expected_netloc
@@ -83,10 +84,7 @@ class Stream:
         self,
         stream_id: int,
         request: Request,
-        protocol: "H2ClientProtocol",
-        download_maxsize: int = 0,
-        download_warnsize: int = 0,
-        fail_on_data_loss: bool = True
+        protocol: "H2ClientProtocol"
     ) -> None:
         """
         Arguments:
@@ -98,9 +96,14 @@ class Stream:
         self._request: Request = request
         self._protocol: "H2ClientProtocol" = protocol
 
-        self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
-        self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
-        self._fail_on_dataloss = self._request.meta.get('download_fail_on_dataloss', fail_on_data_loss)
+        self._download_maxsize = self._request.meta.get(
+            'download_maxsize',
+            self._protocol.metadata['default_download_maxsize']
+        )
+        self._download_warnsize = self._request.meta.get(
+            'download_warnsize',
+            self._protocol.metadata['default_download_warnsize']
+        )
 
         self.request_start_time = None
 
@@ -174,9 +177,9 @@ class Stream:
         # Make sure that we are sending the request to the correct URL
         url = urlparse(self._request.url)
         return (
-            url.netloc == self._protocol.metadata['hostname']
-            or url.netloc == f'{self._protocol.metadata["hostname"]}:{self._protocol.metadata["port"]}'
-            or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}'
+            url.netloc == to_unicode(self._protocol.metadata['uri'].host)
+            or url.netloc == to_unicode(self._protocol.metadata['uri'].netloc)
+            or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
         )
 
     def _get_request_headers(self) -> List[Tuple[str, str]]:
@@ -391,8 +394,8 @@ class Stream:
         elif reason is StreamCloseReason.INVALID_HOSTNAME:
             self._deferred_response.errback(InvalidHostname(
                 self._request,
-                self._protocol.metadata['hostname'],
-                f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}'
+                to_unicode(self._protocol.metadata['uri'].host),
+                f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
             ))
 
     def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None:

scrapy/core/http2/types.py

@@ -3,6 +3,7 @@ from ipaddress import IPv4Address, IPv6Address
 from typing import Union, Optional
 
 from twisted.internet.ssl import Certificate
+from twisted.web.client import URI
 # for python < 3.8 -- typing.TypedDict is undefined
 from typing_extensions import TypedDict
 
@@ -19,14 +20,16 @@ class H2ConnectionMetadataDict(TypedDict):
     # is updated when HTTP/2 connection is  made successfully
     ip_address: Optional[Union[IPv4Address, IPv6Address]]
 
-    # Name of the peer HTTP/2 connection is established
-    hostname: Optional[str]
+    # URI of the peer HTTP/2 connection is made
+    uri: URI
 
-    port: Optional[int]
-
-    # Both ip_address and hostname are used by the Stream before
+    # Both ip_address and uri are used by the Stream before
     # initiating the request to verify that the base address
 
+    # Variables taken from Project Settings
+    default_download_maxsize: int
+    default_download_warnsize: int
+
 
 class H2ResponseDict(TypedDict):
     # Data received frame by frame from the server is appended

tests/test_http2_client_protocol.py

@@ -11,17 +11,18 @@ from h2.exceptions import InvalidBodyLengthError
 from twisted.internet import reactor
 from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError
 from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
-from twisted.internet.protocol import Factory
 from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate
 from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase
+from twisted.web.client import URI
 from twisted.web.http import Request as TxRequest
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.static import File
 
-from scrapy.core.http2.protocol import H2ClientProtocol
+from scrapy.core.http2.protocol import H2ClientFactory
 from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname
 from scrapy.http import Request, Response, JsonRequest
+from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes, to_unicode
 from tests.mockserver import ssl_context_factory, LeafResource, Status
 
@@ -183,7 +184,8 @@ class Https2ClientProtocolTestCase(TestCase):
             trustRoot=self.client_certificate,
             acceptableProtocols=[b'h2']
         )
-        h2_client_factory = Factory.forProtocol(H2ClientProtocol)
+        uri = URI.fromBytes(to_bytes(self.get_url('/')))
+        h2_client_factory = H2ClientFactory(uri, Settings())
         client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
         self.client = yield client_endpoint.connect(h2_client_factory)