diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 2f177656d..55dbcabec 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -20,7 +20,6 @@ from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request - logger = logging.getLogger(__name__) @@ -30,7 +29,8 @@ class H2ClientProtocol(Protocol): self.conn = H2Connection(config=config) # ID of the next request stream - # Following the convention made by hyper-h2 all IDs will be odd + # Following the convention - 'Streams initiated by a client MUST + # use odd-numbered stream identifiers' (RFC 7540) self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs @@ -160,20 +160,25 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ + errors = [] + if not reason.check(connectionDone): + logger.warning("Connection lost with reason " + str(reason)) + errors.append(reason) + + if self._protocol_error: + errors.append(self._protocol_error) + for stream in self.streams.values(): if stream.request_sent: - stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error, from_protocol=True) + stream.close(StreamCloseReason.CONNECTION_LOST, errors, from_protocol=True) else: stream.close(StreamCloseReason.INACTIVE, from_protocol=True) self._active_streams -= len(self.streams) self.streams.clear() - self._send_pending_requests() + self._pending_request_stream_pool.clear() self.conn.close_connection() - if not reason.check(connectionDone): - logger.warning("Connection lost with reason " + str(reason)) - def _handle_events(self, events: list) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 77cfbcfbf..8b66d4b85 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -17,11 +17,9 @@ from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes - if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol - logger = logging.getLogger(__name__) @@ -326,7 +324,12 @@ class Stream: return expected_size != received_body_size - def close(self, reason: StreamCloseReason, error: Optional[Exception] = None, from_protocol: bool = False) -> None: + def close( + self, + reason: StreamCloseReason, + errors: Optional[List[Exception]] = None, + from_protocol: bool = False + ) -> None: """Based on the reason sent we will handle each case. """ if self.stream_closed_server: @@ -374,11 +377,14 @@ class Stream: self._response['headers'][':status'] = '499' self._fire_response_deferred() - elif reason in (StreamCloseReason.RESET, StreamCloseReason.CONNECTION_LOST): + elif reason is StreamCloseReason.RESET: self._deferred_response.errback(ResponseFailed([ - error if error else Failure() + Failure(f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM') ])) + elif reason is StreamCloseReason.CONNECTION_LOST: + self._deferred_response.errback(ResponseFailed(errors)) + elif reason is StreamCloseReason.INACTIVE: self._deferred_response.errback(InactiveStreamClosed(self._request)) @@ -403,7 +409,7 @@ class Stream: response = response_cls( url=self._request.url, - status=self._response['headers'][':status'], + status=int(self._response['headers'][':status']), headers=self._response['headers'], body=body, request=self._request, diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 4e2671478..9d59fdd68 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -46,15 +46,15 @@ DEFAULT_LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'loggers': { + 'hpack': { + 'level': 'ERROR', + }, 'scrapy': { 'level': 'DEBUG', }, 'twisted': { 'level': 'ERROR', }, - 'hpack': { - 'level': 'ERROR', - }, } } diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 98dc98a0f..9efca5267 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -474,7 +474,7 @@ class Https2ClientProtocolTestCase(TestCase): # Close the connection now to fire all the extra 10 requests errback # with InactiveStreamClosed - self.client.transport.abortConnection() + self.client.transport.loseConnection() return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) diff --git a/tox.ini b/tox.ini index bc6314a2f..be3b56e2d 100644 --- a/tox.ini +++ b/tox.ini @@ -83,8 +83,8 @@ deps = -rtests/requirements-py3.txt # Extras botocore==1.3.23 - Pillow==3.4.2 h2==3.2.0 + Pillow==3.4.2 [testenv:extra-deps] deps =