fix: handle CONNECTION_LOST & RESET separately

scrapy/core/http2/protocol.py

@@ -20,7 +20,6 @@ from scrapy.core.http2.stream import Stream, StreamCloseReason
 from scrapy.core.http2.types import H2ConnectionMetadataDict
 from scrapy.http import Request
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -30,7 +29,8 @@ class H2ClientProtocol(Protocol):
         self.conn = H2Connection(config=config)
 
         # ID of the next request stream
-        # Following the convention made by hyper-h2 all IDs will be odd
+        # Following the convention - 'Streams initiated by a client MUST
+        # use odd-numbered stream identifiers' (RFC 7540)
         self._stream_id_generator = itertools.count(start=1, step=2)
 
         # Streams are stored in a dictionary keyed off their stream IDs
@@ -160,20 +160,25 @@ class H2ClientProtocol(Protocol):
         """Called by Twisted when the transport connection is lost.
         No need to write anything to transport here.
         """
+        errors = []
+        if not reason.check(connectionDone):
+            logger.warning("Connection lost with reason " + str(reason))
+            errors.append(reason)
+
+        if self._protocol_error:
+            errors.append(self._protocol_error)
+
         for stream in self.streams.values():
             if stream.request_sent:
-                stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error, from_protocol=True)
+                stream.close(StreamCloseReason.CONNECTION_LOST, errors, from_protocol=True)
             else:
                 stream.close(StreamCloseReason.INACTIVE, from_protocol=True)
 
         self._active_streams -= len(self.streams)
         self.streams.clear()
-        self._send_pending_requests()
+        self._pending_request_stream_pool.clear()
         self.conn.close_connection()
 
-        if not reason.check(connectionDone):
-            logger.warning("Connection lost with reason " + str(reason))
-
     def _handle_events(self, events: list) -> None:
         """Private method which acts as a bridge between the events
         received from the HTTP/2 data and IH2EventsHandler

scrapy/core/http2/stream.py

@@ -17,11 +17,9 @@ from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
 
-
 if TYPE_CHECKING:
     from scrapy.core.http2.protocol import H2ClientProtocol
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -326,7 +324,12 @@ class Stream:
 
         return expected_size != received_body_size
 
-    def close(self, reason: StreamCloseReason, error: Optional[Exception] = None, from_protocol: bool = False) -> None:
+    def close(
+        self,
+        reason: StreamCloseReason,
+        errors: Optional[List[Exception]] = None,
+        from_protocol: bool = False
+    ) -> None:
         """Based on the reason sent we will handle each case.
         """
         if self.stream_closed_server:
@@ -374,11 +377,14 @@ class Stream:
             self._response['headers'][':status'] = '499'
             self._fire_response_deferred()
 
-        elif reason in (StreamCloseReason.RESET, StreamCloseReason.CONNECTION_LOST):
+        elif reason is StreamCloseReason.RESET:
             self._deferred_response.errback(ResponseFailed([
-                error if error else Failure()
+                Failure(f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM')
             ]))
 
+        elif reason is StreamCloseReason.CONNECTION_LOST:
+            self._deferred_response.errback(ResponseFailed(errors))
+
         elif reason is StreamCloseReason.INACTIVE:
             self._deferred_response.errback(InactiveStreamClosed(self._request))
 
@@ -403,7 +409,7 @@ class Stream:
 
         response = response_cls(
             url=self._request.url,
-            status=self._response['headers'][':status'],
+            status=int(self._response['headers'][':status']),
             headers=self._response['headers'],
             body=body,
             request=self._request,

scrapy/utils/log.py

@@ -46,15 +46,15 @@ DEFAULT_LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
     'loggers': {
+        'hpack': {
+            'level': 'ERROR',
+        },
         'scrapy': {
             'level': 'DEBUG',
         },
         'twisted': {
             'level': 'ERROR',
         },
-        'hpack': {
-            'level': 'ERROR',
-        },
     }
 }
 

tests/test_http2_client_protocol.py

@@ -474,7 +474,7 @@ class Https2ClientProtocolTestCase(TestCase):
 
         # Close the connection now to fire all the extra 10 requests errback
         # with InactiveStreamClosed
-        self.client.transport.abortConnection()
+        self.client.transport.loseConnection()
 
         return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True)
 

tox.ini

@@ -83,8 +83,8 @@ deps =
     -rtests/requirements-py3.txt
     # Extras
     botocore==1.3.23
-    Pillow==3.4.2
     h2==3.2.0
+    Pillow==3.4.2
 
 [testenv:extra-deps]
 deps =