From 120007c0577f819a26ff795ac8b9cd6fc397df19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 13:53:47 +0100 Subject: [PATCH 001/114] Add a FAQ entry on how to deal with long lists of allowed domains --- docs/faq.rst | 35 +++++++++++++++++++++++++++++++ docs/topics/spider-middleware.rst | 2 ++ 2 files changed, 37 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 7a0628f88..f56d26c0a 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -149,6 +149,41 @@ How can I make Scrapy consume less memory? See previous question. +How can I prevent memory errors due to many allowed domains? +------------------------------------------------------------ + +If you have a spider with a long list of +:attr:`~scrapy.spiders.Spider.allowed_domains` (e.g. 50,000+), consider +replacing the default +:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware +with a :ref:`custom spider middleware ` that requires +less memory. For example: + +- If your domain names are similar enough, use your own regular expression + instead joining the strings in + :attr:`~scrapy.spiders.Spider.allowed_domains` into a complex regular + expression. + +- If you can `meet the installation requirements`_, use pyre2_ instead of + Python’s re_ to compile your URL-filtering regular expression. See + :issue:`1908`. + +See also other suggestions at `StackOverflow`_. + +.. note:: Remember to disable + :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable + your custom implementation:: + + SPIDER_MIDDLEWARES = { + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None, + 'myproject.middlewares.CustomOffsiteMiddleware': 500, + } + +.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation +.. _pyre2: https://github.com/andreasvc/pyre2 +.. _re: https://docs.python.org/library/re.html +.. _StackOverflow: https://stackoverflow.com/q/36440681/939364 + Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2b7e42771..80357a987 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -54,6 +54,8 @@ value. For example, if you want to disable the off-site middleware:: Finally, keep in mind that some middlewares may need to be enabled through a particular setting. See each middleware documentation for more info. +.. _custom-spider-middleware: + Writing your own spider middleware ================================== From 9408c77a1e16df89feeab055d3530ebad66555e1 Mon Sep 17 00:00:00 2001 From: Aditya Kumar Date: Sun, 31 May 2020 13:09:56 +0530 Subject: [PATCH 002/114] feat(http2): IH2EventsHandler, http2 module --- scrapy/core/http2/__init__.py | 0 scrapy/core/http2/protocol.py | 136 ++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 scrapy/core/http2/__init__.py create mode 100644 scrapy/core/http2/protocol.py diff --git a/scrapy/core/http2/__init__.py b/scrapy/core/http2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py new file mode 100644 index 000000000..4a465168f --- /dev/null +++ b/scrapy/core/http2/protocol.py @@ -0,0 +1,136 @@ +from h2.connection import H2Connection +from h2.config import H2Configuration +from h2.events import ( + ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, + StreamReset, TrailersReceived, WindowUpdated +) + +from scrapy.http import Request + +from twisted.internet.defer import maybeDeferred +from twisted.internet.protocol import Protocol + +from urllib.parse import urlparse + +from zope.interface import implementer, Interface + + +class IH2EventsHandler(Interface): + def connection_terminated(event: ConnectionTerminated): + pass + + def data_received(event: DataReceived): + pass + + def response_received(event: ResponseReceived): + pass + + def stream_ended(event: StreamEnded): + pass + + def stream_reset(event: StreamReset): + pass + + def trailers_received(event: TrailersReceived): + pass + + def window_updated(event: WindowUpdated): + pass + + +@implementer(IH2EventsHandler) +class H2ClientProtocol(Protocol): + def __init__(self): + config = H2Configuration(client_side=True) + self.conn = H2Connection(config=config) + + # List of ongoing stream id's + self.streams = [] + + def request(self, _request: Request): + url = urlparse(_request.url) + + request_headers = [ + (':method', _request.method), + (':authority', url.netloc), + (':scheme', url.scheme), + (':path', url.path), + ] + + # TODO: Check for user-agent while testing + request_headers += list(_request.headers.items()) + + # TODO: Add support for cookies here + + + + + + + def connectionMade(self): + """Called by Twisted when the connection is established. We can start + sending some data now: we should open with the connection preamble. + """ + self.conn.initiate_connection() + self.transport.write(self.conn.data_to_send()) + + def dataReceived(self, data): + events = self.conn.receive_data(data) + + self._handle_events(events) + + _data = self.conn.data_to_send() + if _data: + self.transport.write(data) + + def connectionLost(self, reason): + """Called by Twisted when the transport connection is lost. + """ + + for stream_id in self.streams: + self.conn.end_stream(stream_id) + + def _handle_events(self, events): + """Private method which acts as a bridge between the events + received from the HTTP/2 data and IH2EventsHandler + + Arguments: + events {list} -- A list of events that the remote peer + triggered by sending data + """ + for event in events: + if isinstance(event, ConnectionTerminated): + self.connection_terminated(event) + elif isinstance(event, DataReceived): + self.data_received(event) + elif isinstance(event, ResponseReceived): + self.response_received(event) + elif isinstance(event, StreamEnded): + self.stream_ended(event) + elif isinstance(event, StreamReset): + self.stream_reset(event) + elif isinstance(event, TrailersReceived): + self.trailers_received(event) + elif isinstance(event, WindowUpdated): + self.window_updated(event) + + def connection_terminated(self, event): + pass + + def data_received(self, event): + pass + + def response_received(self, event): + pass + + def stream_ended(self, event): + pass + + def stream_reset(self, event): + pass + + def trailers_received(self, event): + pass + + def window_updated(self, event): + pass From 791292334e86723a80cfd94b2877a258721f2c93 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 2 Jun 2020 09:13:31 +0530 Subject: [PATCH 003/114] chore(http2): Stream class --- scrapy/core/http2/protocol.py | 75 +++++++++++++++++++++++------------ scrapy/core/http2/stream.py | 44 ++++++++++++++++++++ 2 files changed, 94 insertions(+), 25 deletions(-) create mode 100644 scrapy/core/http2/stream.py diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 4a465168f..b95cb05ae 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -6,8 +6,9 @@ from h2.events import ( ) from scrapy.http import Request +from scrapy.core.http2.stream import Stream -from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import Deferred from twisted.internet.protocol import Protocol from urllib.parse import urlparse @@ -40,31 +41,55 @@ class IH2EventsHandler(Interface): @implementer(IH2EventsHandler) class H2ClientProtocol(Protocol): + # TODO: Check for user-agent while testing + # TODO: Add support for cookies + # TODO: Handle priority updates + def __init__(self): config = H2Configuration(client_side=True) self.conn = H2Connection(config=config) - - # List of ongoing stream id's - self.streams = [] + + # ID of the next request stream + # Assuming each request stream creates a new response stream + # we increment by 2 for each new request stream created + self.next_stream_id = 1 + + # Streams are stored in a dictionary keyed off their stream IDs + self.streams = {} + + def _new_stream(self, headers): + """Instantiates a new Stream object + """ + stream = Stream(self.next_stream_id, headers) + + self.next_stream_id += 2 + + return stream def request(self, _request: Request): + """ + + Arguments: + _request {Request} -- [description] + """ url = urlparse(_request.url) - request_headers = [ - (':method', _request.method), - (':authority', url.netloc), - (':scheme', url.scheme), - (':path', url.path), - ] - - # TODO: Check for user-agent while testing - request_headers += list(_request.headers.items()) - - # TODO: Add support for cookies here + _request[":method"] = _request.method + # TODO: Make authority private class variable instead + # of parsing it from request url all requests to same + # host are multiplexed into one connection & a connection + # can have only 1 host at a time + _request[":authority"] = url.netloc + # TODO: Check if scheme can be 'http' for HTTP/2 ? + _request[":scheme"] = "https" + _request[":path"] = url.path + stream = self._new_stream(_request.headers) + d = stream.get_response() + return d def connectionMade(self): @@ -76,7 +101,6 @@ class H2ClientProtocol(Protocol): def dataReceived(self, data): events = self.conn.receive_data(data) - self._handle_events(events) _data = self.conn.data_to_send() @@ -85,9 +109,10 @@ class H2ClientProtocol(Protocol): def connectionLost(self, reason): """Called by Twisted when the transport connection is lost. - """ + """ - for stream_id in self.streams: + for stream_id in self.streams.keys(): + # TODO: Close each Stream instance in a clean manner self.conn.end_stream(stream_id) def _handle_events(self, events): @@ -114,23 +139,23 @@ class H2ClientProtocol(Protocol): elif isinstance(event, WindowUpdated): self.window_updated(event) - def connection_terminated(self, event): + def connection_terminated(self, event: ConnectionTerminated): pass - def data_received(self, event): + def data_received(self, event: DataReceived): pass - def response_received(self, event): + def response_received(self, event: ResponseReceived): pass - def stream_ended(self, event): + def stream_ended(self, event: StreamEnded): pass - def stream_reset(self, event): + def stream_reset(self, event: StreamReset): pass - def trailers_received(self, event): + def trailers_received(self, event: TrailersReceived): pass - def window_updated(self, event): + def window_updated(self, event: WindowUpdated): pass diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py new file mode 100644 index 000000000..558a6552c --- /dev/null +++ b/scrapy/core/http2/stream.py @@ -0,0 +1,44 @@ +from scrapy.http.headers import Headers + + +class Stream: + """Represents a single HTTP/2 Stream. + + Stream is a bidirectional flow of bytes within an established connection, + which may carry one or more messages. Handles the tranfer of HTTP Headers + and Data frames. + """ + + def __init__(self, stream_id, headers): + """ + Arguments: + stream_id {int} -- For one HTTP/2 connection each stream is + uniquely identified by a single integer + headers {Headers} -- HTTP request headers + """ + + # Headers received after sending the request + self.response_headers = Headers({}) + + # Headers which are send with the request + # These cannot be modified any furthur + self._request_headers = headers + + # TODO: Add canceller for the Deferred below + self._deferred_response = Deferred() + + def get_response(self): + """Simply return a Deferred which fires when response + from the asynchronous request is available + + Returns: + Deferred -- Calls the callback when the response is + avaialble + """ + return self._deferred_response + + def receive_data(self, data): + pass + + def receive_headers(self, headers): + pass From 9ff9caecadf8215ce75b0ad4231b8289bca168fe Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 7 Jun 2020 14:04:53 +0530 Subject: [PATCH 004/114] feat(http2): support for GET requests --- scrapy/core/http2/protocol.py | 126 ++++++++++++++++++++++------------ scrapy/core/http2/stream.py | 71 +++++++++++++++---- 2 files changed, 139 insertions(+), 58 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index b95cb05ae..847e74f97 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,20 +1,19 @@ -from h2.connection import H2Connection +import logging + from h2.config import H2Configuration +from h2.connection import H2Connection from h2.events import ( - ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, - StreamReset, TrailersReceived, WindowUpdated + ConnectionTerminated, DataReceived, ResponseReceived, RemoteSettingsChanged, + StreamEnded, StreamReset, TrailersReceived, WindowUpdated ) - -from scrapy.http import Request -from scrapy.core.http2.stream import Stream - -from twisted.internet.defer import Deferred -from twisted.internet.protocol import Protocol - -from urllib.parse import urlparse - +from twisted.internet.protocol import connectionDone, Protocol from zope.interface import implementer, Interface +from scrapy.core.http2.stream import Stream +from scrapy.http import Request + +LOGGER = logging.getLogger(__name__) + class IH2EventsHandler(Interface): def connection_terminated(event: ConnectionTerminated): @@ -26,6 +25,9 @@ class IH2EventsHandler(Interface): def response_received(event: ResponseReceived): pass + def remote_settings_changed(event: RemoteSettingsChanged): + pass + def stream_ended(event: StreamEnded): pass @@ -46,7 +48,7 @@ class H2ClientProtocol(Protocol): # TODO: Handle priority updates def __init__(self): - config = H2Configuration(client_side=True) + config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) # ID of the next request stream @@ -57,60 +59,69 @@ class H2ClientProtocol(Protocol): # Streams are stored in a dictionary keyed off their stream IDs self.streams = {} - def _new_stream(self, headers): + # Boolean to keep track the connection is made + # If requests are received before connection is made + # we keep all requests in a pool and send them as the connection + # is made + self.is_connection_made = False + self._pending_request_stream_pool = [] + + def _new_stream(self, request: Request): """Instantiates a new Stream object """ - stream = Stream(self.next_stream_id, headers) - + stream = Stream(self.next_stream_id, request, self) self.next_stream_id += 2 + self.streams[stream.stream_id] = stream return stream + def _write_to_transport(self): + """ Write data to the underlying transport connection + from the HTTP2 connection instance if any + """ + data = self.conn.data_to_send() + if data: + self.transport.write(data) + def request(self, _request: Request): - """ - - Arguments: - _request {Request} -- [description] - """ - url = urlparse(_request.url) - - _request[":method"] = _request.method - - # TODO: Make authority private class variable instead - # of parsing it from request url all requests to same - # host are multiplexed into one connection & a connection - # can have only 1 host at a time - _request[":authority"] = url.netloc - - # TODO: Check if scheme can be 'http' for HTTP/2 ? - _request[":scheme"] = "https" - _request[":path"] = url.path - - stream = self._new_stream(_request.headers) + stream = self._new_stream(_request) d = stream.get_response() - return d + # If connection is not yet established then add the + # stream to pool or initiate request + if self.is_connection_made: + stream.initiate_request() + else: + self._pending_request_stream_pool.append(stream) + return d def connectionMade(self): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ + LOGGER.info("Connection made to {}".format(self.transport)) self.conn.initiate_connection() - self.transport.write(self.conn.data_to_send()) + self._write_to_transport() + + self.is_connection_made = True + + # Initiate all pending requests + for stream in self._pending_request_stream_pool: + assert isinstance(stream, Stream) + stream.initiate_request() + + self._pending_request_stream_pool.clear() def dataReceived(self, data): events = self.conn.receive_data(data) self._handle_events(events) + self._write_to_transport() - _data = self.conn.data_to_send() - if _data: - self.transport.write(data) + def connectionLost(self, reason=connectionDone): - def connectionLost(self, reason): """Called by Twisted when the transport connection is lost. """ - for stream_id in self.streams.keys(): # TODO: Close each Stream instance in a clean manner self.conn.end_stream(stream_id) @@ -138,18 +149,43 @@ class H2ClientProtocol(Protocol): self.trailers_received(event) elif isinstance(event, WindowUpdated): self.window_updated(event) + elif isinstance(event, RemoteSettingsChanged): + self.remote_settings_changed(event) + + def send_headers(self, stream_id, headers): + """ Send the headers for a given stream to the resource + Initiates a new connection hence. + + Arguments: + stream_id {int} -- Valid stream id + headers {List[Tuple[str, str]]} -- Headers of the request + """ + if stream_id in self.streams: + self.conn.send_headers(stream_id, headers, end_stream=True) + self._write_to_transport() + else: + pass def connection_terminated(self, event: ConnectionTerminated): pass def data_received(self, event: DataReceived): - pass + stream_id = event.stream_id + # TODO: Stream do not exist in self.streams dict + self.streams[stream_id].receive_data(event.data) def response_received(self, event: ResponseReceived): + stream_id = event.stream_id + # TODO: Stream do not exist in self.streams dict + self.streams[stream_id].receive_headers(event.headers) + + def remote_settings_changed(self, event: RemoteSettingsChanged): pass def stream_ended(self, event: StreamEnded): - pass + stream_id = event.stream_id + # TODO: Stream do not exist in self.streams dict + self.streams[stream_id].end_stream() def stream_reset(self, event: StreamReset): pass diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 558a6552c..d2a9f02fa 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,3 +1,8 @@ +from urllib.parse import urlparse + +from twisted.internet.defer import Deferred + +from scrapy.http import Request, Response from scrapy.http.headers import Headers @@ -5,24 +10,30 @@ class Stream: """Represents a single HTTP/2 Stream. Stream is a bidirectional flow of bytes within an established connection, - which may carry one or more messages. Handles the tranfer of HTTP Headers + which may carry one or more messages. Handles the transfer of HTTP Headers and Data frames. + + Role of this class is to + 1. Combine all the data frames """ - def __init__(self, stream_id, headers): + def __init__(self, stream_id: int, request: Request, connection): """ Arguments: stream_id {int} -- For one HTTP/2 connection each stream is uniquely identified by a single integer - headers {Headers} -- HTTP request headers + request {Request} -- HTTP request + connection {H2ClientProtocol} -- HTTP/2 connection this stream belongs to """ - # Headers received after sending the request - self.response_headers = Headers({}) + self.stream_id = stream_id + self._request = request + self._conn = connection - # Headers which are send with the request - # These cannot be modified any furthur - self._request_headers = headers + self._response_data = b"" + + # Headers received after sending the request + self._response_headers = Headers({}) # TODO: Add canceller for the Deferred below self._deferred_response = Deferred() @@ -32,13 +43,47 @@ class Stream: from the asynchronous request is available Returns: - Deferred -- Calls the callback when the response is - avaialble + Deferred -- Calls the callback passing the response """ return self._deferred_response - def receive_data(self, data): - pass + def initiate_request(self): + http2_request_headers = [] + for name, value in self._request.headers.items(): + http2_request_headers.append((name, value)) + + url = urlparse(self._request.url) + http2_request_headers += [ + (":method", self._request.method), + (":authority", url.netloc), + + # TODO: Check if scheme can be "http" for HTTP/2 ? + (":scheme", "https"), + (":path", url.path) + ] + + self._conn.send_headers(self.stream_id, http2_request_headers) + + def receive_data(self, data: bytes): + self._response_data += data def receive_headers(self, headers): - pass + for name, value in headers: + self._response_headers[name] = value + + def end_stream(self): + """Stream is ended by the resource hence no further + data or headers should be expected on this stream. + + We will call the response deferred callback passing + the response object + """ + # TODO: Set flags, certificate, ip_address + response = Response( + url=self._request.url, + status=self._response_headers[":status"], + headers=self._response_headers, + body=self._response_data, + request=self._request + ) + self._deferred_response.callback(response) From d09ccf8d3b2932d9393e6ce4a20c46befdd43acf Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 13 Jun 2020 20:40:01 +0530 Subject: [PATCH 005/114] feat(http2): support for POST requests BREAKING CHANGES - Request is sent successfully with its Response received as well. However, the StreamEnded event is not received which do not fires the response deferred --- scrapy/core/http2/protocol.py | 108 +++++++++++++++++----------------- scrapy/core/http2/stream.py | 79 +++++++++++++++++++++++-- 2 files changed, 127 insertions(+), 60 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 847e74f97..6d926b100 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,4 +1,5 @@ import logging +from typing import Dict, List from h2.config import H2Configuration from h2.connection import H2Connection @@ -7,45 +8,19 @@ from h2.events import ( StreamEnded, StreamReset, TrailersReceived, WindowUpdated ) from twisted.internet.protocol import connectionDone, Protocol -from zope.interface import implementer, Interface from scrapy.core.http2.stream import Stream from scrapy.http import Request LOGGER = logging.getLogger(__name__) +LOGGER.debug = print -class IH2EventsHandler(Interface): - def connection_terminated(event: ConnectionTerminated): - pass - - def data_received(event: DataReceived): - pass - - def response_received(event: ResponseReceived): - pass - - def remote_settings_changed(event: RemoteSettingsChanged): - pass - - def stream_ended(event: StreamEnded): - pass - - def stream_reset(event: StreamReset): - pass - - def trailers_received(event: TrailersReceived): - pass - - def window_updated(event: WindowUpdated): - pass - - -@implementer(IH2EventsHandler) class H2ClientProtocol(Protocol): - # TODO: Check for user-agent while testing - # TODO: Add support for cookies - # TODO: Handle priority updates + # TODO: + # 1. Check for user-agent while testing + # 2. Add support for cookies + # 3. Handle priority updates def __init__(self): config = H2Configuration(client_side=True, header_encoding='utf-8') @@ -57,14 +32,14 @@ class H2ClientProtocol(Protocol): self.next_stream_id = 1 # Streams are stored in a dictionary keyed off their stream IDs - self.streams = {} + self.streams: Dict[int, Stream] = {} # Boolean to keep track the connection is made # If requests are received before connection is made # we keep all requests in a pool and send them as the connection # is made self.is_connection_made = False - self._pending_request_stream_pool = [] + self._pending_request_stream_pool: List[Stream] = [] def _new_stream(self, request: Request): """Instantiates a new Stream object @@ -100,29 +75,27 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ - LOGGER.info("Connection made to {}".format(self.transport)) + LOGGER.debug("Connection made to {}".format(self.transport)) self.conn.initiate_connection() self._write_to_transport() self.is_connection_made = True - # Initiate all pending requests - for stream in self._pending_request_stream_pool: - assert isinstance(stream, Stream) - stream.initiate_request() - - self._pending_request_stream_pool.clear() - def dataReceived(self, data): events = self.conn.receive_data(data) self._handle_events(events) self._write_to_transport() def connectionLost(self, reason=connectionDone): - """Called by Twisted when the transport connection is lost. """ - for stream_id in self.streams.keys(): + LOGGER.debug(f"connectionLost {reason}") + stream_ids = list(self.streams.keys()) + + for stream in self._pending_request_stream_pool: + stream_ids.remove(stream.stream_id) + + for stream_id in stream_ids: # TODO: Close each Stream instance in a clean manner self.conn.end_stream(stream_id) @@ -135,6 +108,7 @@ class H2ClientProtocol(Protocol): triggered by sending data """ for event in events: + LOGGER.debug(event) if isinstance(event, ConnectionTerminated): self.connection_terminated(event) elif isinstance(event, DataReceived): @@ -153,38 +127,62 @@ class H2ClientProtocol(Protocol): self.remote_settings_changed(event) def send_headers(self, stream_id, headers): - """ Send the headers for a given stream to the resource + """Send the headers for a given stream to the resource Initiates a new connection hence. + This function is wrapper for :func:`~h2.connection.H2Connection.send_headers` Arguments: stream_id {int} -- Valid stream id headers {List[Tuple[str, str]]} -- Headers of the request """ - if stream_id in self.streams: - self.conn.send_headers(stream_id, headers, end_stream=True) - self._write_to_transport() - else: - pass + LOGGER.debug(f'Send Headers: stream_id={stream_id} headers={headers}') + self.conn.send_headers(stream_id, headers, end_stream=False) + def send_data(self, stream_id, data): + """Send the data for a given stream to the resource. + Requires request headers to be sent at least once before this + function is called. + This function is wrapper for :func:`~h2.connection.H2Connection.send_data` + + Arguments: + stream_id {int} -- Valid stream id + data {bytes} -- The data to send on the stream. + """ + LOGGER.debug(f"Send Data: stream_id={stream_id} data={data}") + self.conn.send_data(stream_id, data, end_stream=False) + + def end_stream(self, stream_id): + """End the given stream. + This function is wrapper for :func:`~h2.connection.H2Connection.end_stream` + + Arguments: + stream_id {int} - Valid stream id + """ + LOGGER.debug(f"End Stream: stream_id={stream_id}") + self.conn.end_stream(stream_id) + + # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated): pass def data_received(self, event: DataReceived): stream_id = event.stream_id - # TODO: Stream do not exist in self.streams dict self.streams[stream_id].receive_data(event.data) def response_received(self, event: ResponseReceived): stream_id = event.stream_id - # TODO: Stream do not exist in self.streams dict self.streams[stream_id].receive_headers(event.headers) def remote_settings_changed(self, event: RemoteSettingsChanged): - pass + # TODO: handle MAX_CONCURRENT_STREAMS + # Initiate all pending requests + for stream in self._pending_request_stream_pool: + stream.initiate_request() + + self._pending_request_stream_pool.clear() def stream_ended(self, event: StreamEnded): stream_id = event.stream_id - # TODO: Stream do not exist in self.streams dict self.streams[stream_id].end_stream() def stream_reset(self, event: StreamReset): @@ -194,4 +192,6 @@ class H2ClientProtocol(Protocol): pass def window_updated(self, event: WindowUpdated): - pass + stream_id = event.stream_id + if stream_id != 0: + self.streams[stream_id].window_updated() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index d2a9f02fa..ab5a0fc88 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -23,13 +23,23 @@ class Stream: stream_id {int} -- For one HTTP/2 connection each stream is uniquely identified by a single integer request {Request} -- HTTP request - connection {H2ClientProtocol} -- HTTP/2 connection this stream belongs to + connection {H2Connection} -- HTTP/2 connection this stream belongs to. """ - self.stream_id = stream_id self._request = request - self._conn = connection + self._client_protocol = connection + self._request_body = self._request.body + self.content_length = 0 if self._request_body is None else len(self._request_body) + + # Each time we send a data frame, we will decrease value by the amount send. + self.remaining_content_length = self.content_length + + # Flag to keep track whether we have ended this stream + self.stream_ended = True + + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. self._response_data = b"" # Headers received after sending the request @@ -59,10 +69,67 @@ class Stream: # TODO: Check if scheme can be "http" for HTTP/2 ? (":scheme", "https"), - (":path", url.path) + (":path", url.path), + # ("Content-Length", str(self.content_length)) + + # TODO: Make sure 'Content-Type' and 'Content-Encoding' headers + # are sent for request having body ] - self._conn.send_headers(self.stream_id, http2_request_headers) + self._client_protocol.send_headers(self.stream_id, http2_request_headers) + self.send_data() + + def send_data(self): + """Called immediately after the headers are sent. Here we send all the + data as part of the request. + + If the content length is 0 initially then we end the stream immediately and + wait for response data. + """ + + # TODO: + # 1. Add test for sending very large data + # 2. Add test for small data + # 3. Both (1) and (2) should be tested for + # 3.1 Large number of request + # 3.2 Small number of requests + + # Firstly, check what the flow control window is for current stream. + window_size = self._client_protocol.conn.local_flow_control_window(stream_id=self.stream_id) + + # Next, check what the maximum frame size is. + max_frame_size = self._client_protocol.conn.max_outbound_frame_size + + # We will send no more than the window size or the remaining file size + # of data in this call, whichever is smaller. + bytes_to_send = min(window_size, self.remaining_content_length) + + # We now need to send a number of data frames. + while bytes_to_send > 0: + chunk_size = min(bytes_to_send, max_frame_size) + + data_chunk_start = self.content_length - self.remaining_content_length + data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size] + + self._client_protocol.send_data(self.stream_id, data_chunk, end_stream=False) + + bytes_to_send = max(0, bytes_to_send - chunk_size) + self.remaining_content_length = max(0, self.remaining_content_length - chunk_size) + + # End the stream if no more data has to be send + if self.remaining_content_length == 0: + self._client_protocol.end_stream(self.stream_id) + else: + # TODO: Continue from here :) + pass + + def window_updated(self): + """Flow control window size was changed. + Send data that earlier could not be sent as we were + blocked behind the flow control. + """ + if self.remaining_content_length > 0 and not self.stream_ended: + self.send_data() def receive_data(self, data: bytes): self._response_data += data @@ -72,7 +139,7 @@ class Stream: self._response_headers[name] = value def end_stream(self): - """Stream is ended by the resource hence no further + """Stream is ended by the server hence no further data or headers should be expected on this stream. We will call the response deferred callback passing From d06bb12e351f61ee20c28626b003f4b59fd689f7 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 13 Jun 2020 22:29:16 +0530 Subject: [PATCH 006/114] refactor: move H2Connection instance to stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove all wrapper funtions made such that stream can send header/data to H2Connection as they were not necessary BREAKING CHANGES Looks like, for small set of response data the StreamEnded event is emitted and everything works well -- tested for both GET & POST request. Maybe some issue with window size and/or flow control as when the response data needs to be broken into separate chunks -- not all chunks are received everytime which leads to indefinite waiting for next data chunk and the connection is lost due to timeout. 😥 Working on setting up testing environment now. After testing is setup I'll debug the above bug furthur. --- scrapy/core/http2/protocol.py | 74 ++++++++--------------------------- scrapy/core/http2/stream.py | 16 ++++---- 2 files changed, 24 insertions(+), 66 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 6d926b100..4036dfb3e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -4,8 +4,7 @@ from typing import Dict, List from h2.config import H2Configuration from h2.connection import H2Connection from h2.events import ( - ConnectionTerminated, DataReceived, ResponseReceived, RemoteSettingsChanged, - StreamEnded, StreamReset, TrailersReceived, WindowUpdated + ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, StreamReset, WindowUpdated ) from twisted.internet.protocol import connectionDone, Protocol @@ -13,7 +12,6 @@ from scrapy.core.http2.stream import Stream from scrapy.http import Request LOGGER = logging.getLogger(__name__) -LOGGER.debug = print class H2ClientProtocol(Protocol): @@ -44,19 +42,27 @@ class H2ClientProtocol(Protocol): def _new_stream(self, request: Request): """Instantiates a new Stream object """ - stream = Stream(self.next_stream_id, request, self) + stream = Stream(self.next_stream_id, request, self.conn) self.next_stream_id += 2 self.streams[stream.stream_id] = stream return stream + def _send_pending_requests(self): + # TODO: handle MAX_CONCURRENT_STREAMS + # Initiate all pending requests + for stream in self._pending_request_stream_pool: + stream.initiate_request() + self._write_to_transport() + + self._pending_request_stream_pool.clear() + def _write_to_transport(self): """ Write data to the underlying transport connection from the HTTP2 connection instance if any """ data = self.conn.data_to_send() - if data: - self.transport.write(data) + self.transport.write(data) def request(self, _request: Request): stream = self._new_stream(_request) @@ -75,10 +81,11 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ - LOGGER.debug("Connection made to {}".format(self.transport)) self.conn.initiate_connection() self._write_to_transport() + self._send_pending_requests() + self.is_connection_made = True def dataReceived(self, data): @@ -89,7 +96,6 @@ class H2ClientProtocol(Protocol): def connectionLost(self, reason=connectionDone): """Called by Twisted when the transport connection is lost. """ - LOGGER.debug(f"connectionLost {reason}") stream_ids = list(self.streams.keys()) for stream in self._pending_request_stream_pool: @@ -119,47 +125,10 @@ class H2ClientProtocol(Protocol): self.stream_ended(event) elif isinstance(event, StreamReset): self.stream_reset(event) - elif isinstance(event, TrailersReceived): - self.trailers_received(event) elif isinstance(event, WindowUpdated): self.window_updated(event) - elif isinstance(event, RemoteSettingsChanged): - self.remote_settings_changed(event) - - def send_headers(self, stream_id, headers): - """Send the headers for a given stream to the resource - Initiates a new connection hence. - This function is wrapper for :func:`~h2.connection.H2Connection.send_headers` - - Arguments: - stream_id {int} -- Valid stream id - headers {List[Tuple[str, str]]} -- Headers of the request - """ - LOGGER.debug(f'Send Headers: stream_id={stream_id} headers={headers}') - self.conn.send_headers(stream_id, headers, end_stream=False) - - def send_data(self, stream_id, data): - """Send the data for a given stream to the resource. - Requires request headers to be sent at least once before this - function is called. - This function is wrapper for :func:`~h2.connection.H2Connection.send_data` - - Arguments: - stream_id {int} -- Valid stream id - data {bytes} -- The data to send on the stream. - """ - LOGGER.debug(f"Send Data: stream_id={stream_id} data={data}") - self.conn.send_data(stream_id, data, end_stream=False) - - def end_stream(self, stream_id): - """End the given stream. - This function is wrapper for :func:`~h2.connection.H2Connection.end_stream` - - Arguments: - stream_id {int} - Valid stream id - """ - LOGGER.debug(f"End Stream: stream_id={stream_id}") - self.conn.end_stream(stream_id) + else: + LOGGER.info("Received unhandled event {}".format(event)) # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated): @@ -173,14 +142,6 @@ class H2ClientProtocol(Protocol): stream_id = event.stream_id self.streams[stream_id].receive_headers(event.headers) - def remote_settings_changed(self, event: RemoteSettingsChanged): - # TODO: handle MAX_CONCURRENT_STREAMS - # Initiate all pending requests - for stream in self._pending_request_stream_pool: - stream.initiate_request() - - self._pending_request_stream_pool.clear() - def stream_ended(self, event: StreamEnded): stream_id = event.stream_id self.streams[stream_id].end_stream() @@ -188,9 +149,6 @@ class H2ClientProtocol(Protocol): def stream_reset(self, event: StreamReset): pass - def trailers_received(self, event: TrailersReceived): - pass - def window_updated(self, event: WindowUpdated): stream_id = event.stream_id if stream_id != 0: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index ab5a0fc88..b37755042 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,5 +1,6 @@ from urllib.parse import urlparse +from h2.connection import H2Connection from twisted.internet.defer import Deferred from scrapy.http import Request, Response @@ -17,7 +18,7 @@ class Stream: 1. Combine all the data frames """ - def __init__(self, stream_id: int, request: Request, connection): + def __init__(self, stream_id: int, request: Request, connection: H2Connection): """ Arguments: stream_id {int} -- For one HTTP/2 connection each stream is @@ -27,7 +28,7 @@ class Stream: """ self.stream_id = stream_id self._request = request - self._client_protocol = connection + self._conn = connection self._request_body = self._request.body self.content_length = 0 if self._request_body is None else len(self._request_body) @@ -70,13 +71,12 @@ class Stream: # TODO: Check if scheme can be "http" for HTTP/2 ? (":scheme", "https"), (":path", url.path), - # ("Content-Length", str(self.content_length)) # TODO: Make sure 'Content-Type' and 'Content-Encoding' headers # are sent for request having body ] - self._client_protocol.send_headers(self.stream_id, http2_request_headers) + self._conn.send_headers(self.stream_id, http2_request_headers, end_stream=False) self.send_data() def send_data(self): @@ -95,10 +95,10 @@ class Stream: # 3.2 Small number of requests # Firstly, check what the flow control window is for current stream. - window_size = self._client_protocol.conn.local_flow_control_window(stream_id=self.stream_id) + window_size = self._conn.local_flow_control_window(stream_id=self.stream_id) # Next, check what the maximum frame size is. - max_frame_size = self._client_protocol.conn.max_outbound_frame_size + max_frame_size = self._conn.max_outbound_frame_size # We will send no more than the window size or the remaining file size # of data in this call, whichever is smaller. @@ -111,14 +111,14 @@ class Stream: data_chunk_start = self.content_length - self.remaining_content_length data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size] - self._client_protocol.send_data(self.stream_id, data_chunk, end_stream=False) + self._conn.send_data(self.stream_id, data_chunk, end_stream=False) bytes_to_send = max(0, bytes_to_send - chunk_size) self.remaining_content_length = max(0, self.remaining_content_length - chunk_size) # End the stream if no more data has to be send if self.remaining_content_length == 0: - self._client_protocol.end_stream(self.stream_id) + self._conn.end_stream(self.stream_id) else: # TODO: Continue from here :) pass From de4a34365a2d872eb69ab2a2edfb723658e28530 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 14 Jun 2020 22:40:49 +0530 Subject: [PATCH 007/114] fix: large data chunk not received Every data chunk received needs to be acknowledged to - update the flow control window size - get furthur data chunks from the server --- scrapy/core/http2/protocol.py | 74 +++++++++++++++++++++--------- scrapy/core/http2/stream.py | 85 ++++++++++++++++++++++++++++++----- 2 files changed, 129 insertions(+), 30 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 4036dfb3e..167596340 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,10 +1,10 @@ import logging -from typing import Dict, List from h2.config import H2Configuration from h2.connection import H2Connection from h2.events import ( - ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, StreamReset, WindowUpdated + ConnectionTerminated, DataReceived, ResponseReceived, + StreamEnded, StreamReset, WindowUpdated ) from twisted.internet.protocol import connectionDone, Protocol @@ -18,33 +18,57 @@ class H2ClientProtocol(Protocol): # TODO: # 1. Check for user-agent while testing # 2. Add support for cookies - # 3. Handle priority updates + # 3. Handle priority updates (Not required) + # 4. Handle case when received events have StreamID = 0 (applied to H2Connection) + # 1 & 2: + # - Automatically handled by the Request middleware + # - request.headers will have 'Set-Cookie' value def __init__(self): config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) + # Address of the server we are connected to + # these are updated when connection is successfully made + self.destination = None + # ID of the next request stream - # Assuming each request stream creates a new response stream - # we increment by 2 for each new request stream created + # Following the convention made by hyper-h2 each client ID + # will be odd. self.next_stream_id = 1 # Streams are stored in a dictionary keyed off their stream IDs - self.streams: Dict[int, Stream] = {} + self.streams = {} # Boolean to keep track the connection is made # If requests are received before connection is made # we keep all requests in a pool and send them as the connection # is made self.is_connection_made = False - self._pending_request_stream_pool: List[Stream] = [] + self._pending_request_stream_pool = [] + + def _stream_close_cb(self, stream_id: int): + """Called when stream is closed completely + """ + try: + del self.streams[stream_id] + except KeyError: + pass def _new_stream(self, request: Request): """Instantiates a new Stream object """ - stream = Stream(self.next_stream_id, request, self.conn) + stream_id = self.next_stream_id self.next_stream_id += 2 + stream = Stream( + stream_id=stream_id, + request=request, + connection=self.conn, + write_to_transport=self._write_to_transport, + cb_close=lambda: self._stream_close_cb(stream_id) + ) + self.streams[stream.stream_id] = stream return stream @@ -53,7 +77,6 @@ class H2ClientProtocol(Protocol): # Initiate all pending requests for stream in self._pending_request_stream_pool: stream.initiate_request() - self._write_to_transport() self._pending_request_stream_pool.clear() @@ -81,13 +104,15 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ + self.destination = self.transport.connector.getDestination() + LOGGER.info('Connection made to {}'.format(self.destination)) + self.conn.initiate_connection() self._write_to_transport() + self.is_connection_made = True self._send_pending_requests() - self.is_connection_made = True - def dataReceived(self, data): events = self.conn.receive_data(data) self._handle_events(events) @@ -95,15 +120,18 @@ class H2ClientProtocol(Protocol): def connectionLost(self, reason=connectionDone): """Called by Twisted when the transport connection is lost. + No need to write anything to transport here. """ - stream_ids = list(self.streams.keys()) + # Pop all streams which were pending and were not yet started + for stream_id in list(self.streams): + try: + self.streams[stream_id].lost_connection() + except KeyError: + pass - for stream in self._pending_request_stream_pool: - stream_ids.remove(stream.stream_id) + self.conn.close_connection() - for stream_id in stream_ids: - # TODO: Close each Stream instance in a clean manner - self.conn.end_stream(stream_id) + LOGGER.info("Connection lost with reason " + str(reason)) def _handle_events(self, events): """Private method which acts as a bridge between the events @@ -136,7 +164,7 @@ class H2ClientProtocol(Protocol): def data_received(self, event: DataReceived): stream_id = event.stream_id - self.streams[stream_id].receive_data(event.data) + self.streams[stream_id].receive_data(event.data, event.flow_controlled_length) def response_received(self, event: ResponseReceived): stream_id = event.stream_id @@ -147,9 +175,15 @@ class H2ClientProtocol(Protocol): self.streams[stream_id].end_stream() def stream_reset(self, event: StreamReset): - pass + # TODO: event.stream_id was abruptly closed + # Q. What should be the response? (Failure/Partial/???) + self.streams[event.stream_id].reset() def window_updated(self, event: WindowUpdated): stream_id = event.stream_id if stream_id != 0: - self.streams[stream_id].window_updated() + self.streams[stream_id].receive_window_update(event.delta) + else: + # TODO: + # Q. What to do when StreamID=0 ? + pass diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index b37755042..7c9cf7cf5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,3 +1,4 @@ +import logging from urllib.parse import urlparse from h2.connection import H2Connection @@ -6,6 +7,8 @@ from twisted.internet.defer import Deferred from scrapy.http import Request, Response from scrapy.http.headers import Headers +LOGGER = logging.getLogger(__name__) + class Stream: """Represents a single HTTP/2 Stream. @@ -18,17 +21,30 @@ class Stream: 1. Combine all the data frames """ - def __init__(self, stream_id: int, request: Request, connection: H2Connection): + def __init__( + self, + stream_id: int, + request: Request, + connection: H2Connection, + write_to_transport, + cb_close + ): """ Arguments: stream_id {int} -- For one HTTP/2 connection each stream is uniquely identified by a single integer request {Request} -- HTTP request connection {H2Connection} -- HTTP/2 connection this stream belongs to. + write_to_transport {callable} -- Method used to write & send data to the server + This method should be used whenever some frame is to be sent to the server. + cb_close {callable} -- Method called when this stream is closed + to notify the TCP connection instance. """ self.stream_id = stream_id self._request = request self._conn = connection + self._write_to_transport = write_to_transport + self._cb_close = cb_close self._request_body = self._request.body self.content_length = 0 if self._request_body is None else len(self._request_body) @@ -36,13 +52,20 @@ class Stream: # Each time we send a data frame, we will decrease value by the amount send. self.remaining_content_length = self.content_length - # Flag to keep track whether we have ended this stream - self.stream_ended = True + # Flag to keep track whether we have closed this stream + self.stream_closed_local = False + + # Flag to keep track whether the server has closed the stream + self.stream_closed_server = False # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. self._response_data = b"" + # The amount of data received that counts against the flow control + # window + self._response_flow_controlled_size = 0 + # Headers received after sending the request self._response_headers = Headers({}) @@ -77,6 +100,8 @@ class Stream: ] self._conn.send_headers(self.stream_id, http2_request_headers, end_stream=False) + self._write_to_transport() + self.send_data() def send_data(self): @@ -112,32 +137,59 @@ class Stream: data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size] self._conn.send_data(self.stream_id, data_chunk, end_stream=False) + self._write_to_transport() bytes_to_send = max(0, bytes_to_send - chunk_size) self.remaining_content_length = max(0, self.remaining_content_length - chunk_size) # End the stream if no more data has to be send if self.remaining_content_length == 0: + self.stream_closed_local = True self._conn.end_stream(self.stream_id) - else: - # TODO: Continue from here :) - pass - def window_updated(self): + self._write_to_transport() + + # Q. What about the rest of the data? + # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame + + def receive_window_update(self, delta): """Flow control window size was changed. Send data that earlier could not be sent as we were blocked behind the flow control. + + Arguments: + delta -- Window change delta """ - if self.remaining_content_length > 0 and not self.stream_ended: + if self.remaining_content_length > 0 and not self.stream_closed_local: self.send_data() - def receive_data(self, data: bytes): + def receive_data(self, data: bytes, flow_controlled_length: int): self._response_data += data + self._response_flow_controlled_size += flow_controlled_length + + # Acknowledge the data received + self._conn.acknowledge_received_data( + self._response_flow_controlled_size, + self.stream_id + ) def receive_headers(self, headers): for name, value in headers: self._response_headers[name] = value + def reset(self): + """Received a RST_STREAM -- forcefully reset""" + # TODO: + # Q1. Do we need to send the request again? + # Q2. What response should we send now? + self.stream_closed_server = True + self._cb_close() + + def lost_connection(self): + # TODO: Same as self.reset + self.stream_closed_server = True + self._cb_close() + def end_stream(self): """Stream is ended by the server hence no further data or headers should be expected on this stream. @@ -145,7 +197,20 @@ class Stream: We will call the response deferred callback passing the response object """ - # TODO: Set flags, certificate, ip_address + assert self.stream_closed_server is False + self.stream_closed_server = True + + self._fire_response_deferred() + self._cb_close() + + def _fire_response_deferred(self): + # TODO: + # 1. Set flags, certificate, ip_address in response + # 2. Should we fire this in case of + # 2.1 StreamReset in between when data is received partially + # 2.2 Forcefully closed the stream + + # NOTE: Presently on fired with successful response response = Response( url=self._request.url, status=self._response_headers[":status"], From 01ad8b31ab7fe86fd78a70b09a6dd61a497e0ccb Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 15 Jun 2020 05:14:00 +0530 Subject: [PATCH 008/114] refactor(http2): clean up - make separate function to parse http headers from Request instance --- scrapy/core/http2/protocol.py | 12 ++------ scrapy/core/http2/stream.py | 55 ++++++++++++++++++----------------- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 167596340..915284c33 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -50,10 +50,7 @@ class H2ClientProtocol(Protocol): def _stream_close_cb(self, stream_id: int): """Called when stream is closed completely """ - try: - del self.streams[stream_id] - except KeyError: - pass + self.streams.pop(stream_id, None) def _new_stream(self, request: Request): """Instantiates a new Stream object @@ -66,7 +63,7 @@ class H2ClientProtocol(Protocol): request=request, connection=self.conn, write_to_transport=self._write_to_transport, - cb_close=lambda: self._stream_close_cb(stream_id) + cb_close=self._stream_close_cb ) self.streams[stream.stream_id] = stream @@ -124,10 +121,7 @@ class H2ClientProtocol(Protocol): """ # Pop all streams which were pending and were not yet started for stream_id in list(self.streams): - try: - self.streams[stream_id].lost_connection() - except KeyError: - pass + self.streams[stream_id].lost_connection() self.conn.close_connection() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 7c9cf7cf5..a0b75850d 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -81,25 +81,27 @@ class Stream: """ return self._deferred_response - def initiate_request(self): - http2_request_headers = [] - for name, value in self._request.headers.items(): - http2_request_headers.append((name, value)) - + def _get_request_headers(self): url = urlparse(self._request.url) - http2_request_headers += [ - (":method", self._request.method), - (":authority", url.netloc), - # TODO: Check if scheme can be "http" for HTTP/2 ? - (":scheme", "https"), - (":path", url.path), + # Make sure pseudo-headers comes before all the other headers + headers = [ + (':method', self._request.method), + (':authority', url.netloc), - # TODO: Make sure 'Content-Type' and 'Content-Encoding' headers - # are sent for request having body + # TODO: Check if scheme can be 'http' for HTTP/2 ? + (':scheme', 'https'), + (':path', url.path), ] - self._conn.send_headers(self.stream_id, http2_request_headers, end_stream=False) + for name, value in self._request.headers.items(): + headers.append((name, value[0])) + + return headers + + def initiate_request(self): + headers = self._get_request_headers() + self._conn.send_headers(self.stream_id, headers, end_stream=False) self._write_to_transport() self.send_data() @@ -127,23 +129,24 @@ class Stream: # We will send no more than the window size or the remaining file size # of data in this call, whichever is smaller. - bytes_to_send = min(window_size, self.remaining_content_length) + bytes_to_send_size = min(window_size, self.remaining_content_length) # We now need to send a number of data frames. - while bytes_to_send > 0: - chunk_size = min(bytes_to_send, max_frame_size) + while bytes_to_send_size > 0: + chunk_size = min(bytes_to_send_size, max_frame_size) - data_chunk_start = self.content_length - self.remaining_content_length - data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size] + data_chunk_start_id = self.content_length - self.remaining_content_length + data_chunk = self._request_body[data_chunk_start_id:data_chunk_start_id + chunk_size] self._conn.send_data(self.stream_id, data_chunk, end_stream=False) - self._write_to_transport() - bytes_to_send = max(0, bytes_to_send - chunk_size) - self.remaining_content_length = max(0, self.remaining_content_length - chunk_size) + bytes_to_send_size = bytes_to_send_size - chunk_size + self.remaining_content_length = self.remaining_content_length - chunk_size # End the stream if no more data has to be send - if self.remaining_content_length == 0: + if self.remaining_content_length <= 0: + self.remaining_content_length = 0 + self.stream_closed_local = True self._conn.end_stream(self.stream_id) @@ -183,12 +186,12 @@ class Stream: # Q1. Do we need to send the request again? # Q2. What response should we send now? self.stream_closed_server = True - self._cb_close() + self._cb_close(self.stream_id) def lost_connection(self): # TODO: Same as self.reset self.stream_closed_server = True - self._cb_close() + self._cb_close(self.stream_id) def end_stream(self): """Stream is ended by the server hence no further @@ -201,7 +204,7 @@ class Stream: self.stream_closed_server = True self._fire_response_deferred() - self._cb_close() + self._cb_close(self.stream_id) def _fire_response_deferred(self): # TODO: From 089dbc75e78a2da9c455f21bb3c7ebaaeb2e3582 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 17 Jun 2020 20:57:03 +0530 Subject: [PATCH 009/114] chore: use deque for pending request pool - Use itertools.count to generate next stream_id BREAKING CHANGES When sending data/body more than the local flow control window -- no window update occurs to send the remaining data frames. Hence, the complete body is not send resulting in no response received. --- scrapy/core/http2/protocol.py | 16 +++++++++------- scrapy/core/http2/stream.py | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 915284c33..dbc048ffa 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,4 +1,6 @@ +import itertools import logging +from collections import deque from h2.config import H2Configuration from h2.connection import H2Connection @@ -35,7 +37,7 @@ class H2ClientProtocol(Protocol): # ID of the next request stream # Following the convention made by hyper-h2 each client ID # will be odd. - self.next_stream_id = 1 + self.stream_id_count = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs self.streams = {} @@ -45,7 +47,7 @@ class H2ClientProtocol(Protocol): # we keep all requests in a pool and send them as the connection # is made self.is_connection_made = False - self._pending_request_stream_pool = [] + self._pending_request_stream_pool = deque() def _stream_close_cb(self, stream_id: int): """Called when stream is closed completely @@ -55,8 +57,7 @@ class H2ClientProtocol(Protocol): def _new_stream(self, request: Request): """Instantiates a new Stream object """ - stream_id = self.next_stream_id - self.next_stream_id += 2 + stream_id = next(self.stream_id_count) stream = Stream( stream_id=stream_id, @@ -72,11 +73,10 @@ class H2ClientProtocol(Protocol): def _send_pending_requests(self): # TODO: handle MAX_CONCURRENT_STREAMS # Initiate all pending requests - for stream in self._pending_request_stream_pool: + while len(self._pending_request_stream_pool): + stream = self._pending_request_stream_pool.popleft() stream.initiate_request() - self._pending_request_stream_pool.clear() - def _write_to_transport(self): """ Write data to the underlying transport connection from the HTTP2 connection instance if any @@ -108,6 +108,8 @@ class H2ClientProtocol(Protocol): self._write_to_transport() self.is_connection_made = True + # Send off all the pending requests + # as now we have established a proper HTTP/2 connection self._send_pending_requests() def dataReceived(self, data): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a0b75850d..c2e1adce5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -132,7 +132,7 @@ class Stream: bytes_to_send_size = min(window_size, self.remaining_content_length) # We now need to send a number of data frames. - while bytes_to_send_size > 0: + while bytes_to_send_size: chunk_size = min(bytes_to_send_size, max_frame_size) data_chunk_start_id = self.content_length - self.remaining_content_length @@ -163,7 +163,7 @@ class Stream: Arguments: delta -- Window change delta """ - if self.remaining_content_length > 0 and not self.stream_closed_local: + if self.stream_closed_local is False: self.send_data() def receive_data(self, data: bytes, flow_controlled_length: int): From 700df3eeb7c540892c8b2910db688f1cc5d1c845 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 17 Jun 2020 21:02:14 +0530 Subject: [PATCH 010/114] test: mockserver with h2 protocol for tests - add Twisted[http2] in setup.py requirements - add test_protocol.py to test the current implementation BREAKING CHANGES test_download times out because of no protocol negotiated between Mockserver and HTTP/2 client --- scrapy/core/http2/test_protocol.py | 67 +++++++++++++++++++++++++++ setup.py | 6 +-- tests/test_http2_client_protocol.py | 71 +++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 scrapy/core/http2/test_protocol.py create mode 100644 tests/test_http2_client_protocol.py diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.py new file mode 100644 index 000000000..c7782a518 --- /dev/null +++ b/scrapy/core/http2/test_protocol.py @@ -0,0 +1,67 @@ +# This is simple script to test + +import json + +from twisted.internet import reactor +from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint +from twisted.internet.ssl import optionsForClientTLS + +from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.http import Request, Response, JsonRequest + +try: + with open('data.json', 'r') as f: + JSON_DATA = json.load(f) +except: + JSON_DATA = { + "data": "To test for really large amount of data -- Add data.json with lots of data.", + "why": "To test whether correct data is sent :)" + } + +# Use nghttp2 for testing whether basic setup works - for small response +HTTPBIN_AUTHORITY = u'nghttp2.org' +HTTPBIN_REQUEST_URLS = 1 * [ + Request(url='https://nghttp2.org/httpbin/get', method='GET'), + Request(url='https://nghttp2.org/httpbin/post', method='POST'), + JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA), +] + +# Use POKE_API for testing large responses +POKE_API_AUTHORITY = u'pokeapi.co' +POKE_API_REQUESTS = 15 * [ + Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'), + Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'), + Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'), + Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'), # should give 404 +] + +AUTHORITY = POKE_API_AUTHORITY +REQUEST_URLS = POKE_API_REQUESTS + +options = optionsForClientTLS( + hostname=AUTHORITY, + acceptableProtocols=[b'h2'], +) + +protocol = H2ClientProtocol() + +count_responses = 1 + + +def print_response(response): + global count_responses + assert isinstance(response, Response) + print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body))) + count_responses = count_responses + 1 + + +for request in REQUEST_URLS: + d = protocol.request(request) + d.addCallback(print_response) + +connectProtocol( + SSL4ClientEndpoint(reactor, AUTHORITY, 443, options), + protocol +) + +reactor.run() diff --git a/setup.py b/setup.py index 1b3c6771a..dafa5684a 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ from os.path import dirname, join + from pkg_resources import parse_version from setuptools import setup, find_packages, __version__ as setuptools_version - with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: version = f.read().decode('ascii').strip() @@ -25,12 +25,11 @@ if has_environment_marker_platform_impl_support(): 'PyPyDispatcher>=2.1.0', ] - setup( name='Scrapy', version=version, url='https://scrapy.org', - project_urls = { + project_urls={ 'Documentation': 'https://docs.scrapy.org/', 'Source': 'https://github.com/scrapy/scrapy', 'Tracker': 'https://github.com/scrapy/scrapy/issues', @@ -69,6 +68,7 @@ setup( python_requires='>=3.5', install_requires=[ 'Twisted>=17.9.0', + 'Twisted[http2]>=17.9.0' 'cryptography>=2.0', 'cssselect>=0.9.1', 'lxml>=3.5.0', diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py new file mode 100644 index 000000000..7830f7028 --- /dev/null +++ b/tests/test_http2_client_protocol.py @@ -0,0 +1,71 @@ +import os +import shutil + +from twisted.internet import defer, reactor +from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint +from twisted.internet.ssl import optionsForClientTLS +from twisted.protocols.policies import WrappingFactory +from twisted.python.filepath import FilePath +from twisted.trial import unittest +from twisted.web import static, server + +from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.http import Request +from tests.mockserver import ssl_context_factory + + +class Http2ClientProtocolTestCase(unittest.TestCase): + scheme = 'https' + + # only used for HTTPS tests + file_key = 'keys/localhost.key' + file_certificate = 'keys/localhost.crt' + + def setUp(self): + # Start server for testing + self.path_temp = self.mktemp() + os.mkdir(self.path_temp) + FilePath(self.path_temp).child('file').setContent(b"0123456789") + r = static.File(self.path_temp) + + self.site = server.Site(r, timeout=None) + self.wrapper = WrappingFactory(self.site) + self.host = 'localhost' + if self.scheme is 'https': + self.port = reactor.listenSSL( + 0, self.wrapper, + ssl_context_factory(self.file_key, self.file_certificate), + interface=self.host + ) + else: + self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) + + self.port_number = self.port.getHost().port + + # Connect to the server using the custom HTTP2ClientProtocol + options = optionsForClientTLS( + hostname=self.host, + acceptableProtocols=[b'h2'] + ) + + self.protocol = H2ClientProtocol() + + connectProtocol( + endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options), + protocol=self.protocol + ) + + def getURL(self, path): + return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path) + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + shutil.rmtree(self.path_temp) + + def test_download(self): + request = Request(self.getURL('file')) + d = self.protocol.request(request) + d.addCallback(lambda response: response.body) + d.addCallback(self.assertEqual, b"0123456789") + return d From 303485a9b4fd86c5123c96c81a9401b5e323a91d Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 21 Jun 2020 00:33:34 +0530 Subject: [PATCH 011/114] fix(http2): POST request not sending large body --- scrapy/core/http2/protocol.py | 51 +++++++----- scrapy/core/http2/stream.py | 148 ++++++++++++++++++++++++---------- 2 files changed, 138 insertions(+), 61 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index dbc048ffa..d6134183a 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,3 +1,4 @@ +import ipaddress import itertools import logging from collections import deque @@ -5,7 +6,7 @@ from collections import deque from h2.config import H2Configuration from h2.connection import H2Connection from h2.events import ( - ConnectionTerminated, DataReceived, ResponseReceived, + DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) from twisted.internet.protocol import connectionDone, Protocol @@ -49,6 +50,13 @@ class H2ClientProtocol(Protocol): self.is_connection_made = False self._pending_request_stream_pool = deque() + # Some meta data of this connection + # initialized when connection is successfully made + self._metadata = { + 'certificate': None, + 'ip_address': None + } + def _stream_close_cb(self, stream_id: int): """Called when stream is closed completely """ @@ -63,6 +71,7 @@ class H2ClientProtocol(Protocol): stream_id=stream_id, request=request, connection=self.conn, + metadata=self._metadata, write_to_transport=self._write_to_transport, cb_close=self._stream_close_cb ) @@ -73,7 +82,7 @@ class H2ClientProtocol(Protocol): def _send_pending_requests(self): # TODO: handle MAX_CONCURRENT_STREAMS # Initiate all pending requests - while len(self._pending_request_stream_pool): + while self._pending_request_stream_pool: stream = self._pending_request_stream_pool.popleft() stream.initiate_request() @@ -84,6 +93,8 @@ class H2ClientProtocol(Protocol): data = self.conn.data_to_send() self.transport.write(data) + LOGGER.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address'])) + def request(self, _request: Request): stream = self._new_stream(_request) d = stream.get_response() @@ -101,17 +112,16 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ - self.destination = self.transport.connector.getDestination() + self.destination = self.transport.getPeer() LOGGER.info('Connection made to {}'.format(self.destination)) + self._metadata['certificate'] = self.transport.getPeerCertificate() + self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) + self.conn.initiate_connection() self._write_to_transport() self.is_connection_made = True - # Send off all the pending requests - # as now we have established a proper HTTP/2 connection - self._send_pending_requests() - def dataReceived(self, data): events = self.conn.receive_data(data) self._handle_events(events) @@ -123,7 +133,7 @@ class H2ClientProtocol(Protocol): """ # Pop all streams which were pending and were not yet started for stream_id in list(self.streams): - self.streams[stream_id].lost_connection() + self.streams[stream_id].close() self.conn.close_connection() @@ -139,9 +149,7 @@ class H2ClientProtocol(Protocol): """ for event in events: LOGGER.debug(event) - if isinstance(event, ConnectionTerminated): - self.connection_terminated(event) - elif isinstance(event, DataReceived): + if isinstance(event, DataReceived): self.data_received(event) elif isinstance(event, ResponseReceived): self.response_received(event) @@ -151,13 +159,12 @@ class H2ClientProtocol(Protocol): self.stream_reset(event) elif isinstance(event, WindowUpdated): self.window_updated(event) + elif isinstance(event, SettingsAcknowledged): + self.settings_acknowledged(event) else: LOGGER.info("Received unhandled event {}".format(event)) # Event handler functions starts here - def connection_terminated(self, event: ConnectionTerminated): - pass - def data_received(self, event: DataReceived): stream_id = event.stream_id self.streams[stream_id].receive_data(event.data, event.flow_controlled_length) @@ -166,20 +173,26 @@ class H2ClientProtocol(Protocol): stream_id = event.stream_id self.streams[stream_id].receive_headers(event.headers) + def settings_acknowledged(self, event: SettingsAcknowledged): + # Send off all the pending requests + # as now we have established a proper HTTP/2 connection + self._send_pending_requests() + def stream_ended(self, event: StreamEnded): stream_id = event.stream_id - self.streams[stream_id].end_stream() + self.streams[stream_id].close() def stream_reset(self, event: StreamReset): # TODO: event.stream_id was abruptly closed # Q. What should be the response? (Failure/Partial/???) - self.streams[event.stream_id].reset() + self.streams[event.stream_id].close(event) def window_updated(self, event: WindowUpdated): stream_id = event.stream_id if stream_id != 0: self.streams[stream_id].receive_window_update(event.delta) else: - # TODO: - # Q. What to do when StreamID=0 ? - pass + # Send leftover data for all the streams + for stream in self.streams.values(): + if stream.request_sent: + stream.send_data() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index c2e1adce5..e8b4471d6 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,11 +1,15 @@ import logging +from typing import Dict from urllib.parse import urlparse from h2.connection import H2Connection +from h2.events import StreamEnded +from h2.exceptions import StreamClosedError from twisted.internet.defer import Deferred -from scrapy.http import Request, Response +from scrapy.http import Request from scrapy.http.headers import Headers +from scrapy.responsetypes import responsetypes LOGGER = logging.getLogger(__name__) @@ -22,12 +26,13 @@ class Stream: """ def __init__( - self, - stream_id: int, - request: Request, - connection: H2Connection, - write_to_transport, - cb_close + self, + stream_id: int, + request: Request, + connection: H2Connection, + metadata: Dict, + write_to_transport, + cb_close ): """ Arguments: @@ -35,6 +40,7 @@ class Stream: uniquely identified by a single integer request {Request} -- HTTP request connection {H2Connection} -- HTTP/2 connection this stream belongs to. + metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection write_to_transport {callable} -- Method used to write & send data to the server This method should be used whenever some frame is to be sent to the server. cb_close {callable} -- Method called when this stream is closed @@ -43,12 +49,16 @@ class Stream: self.stream_id = stream_id self._request = request self._conn = connection + self._metadata = metadata self._write_to_transport = write_to_transport self._cb_close = cb_close self._request_body = self._request.body self.content_length = 0 if self._request_body is None else len(self._request_body) + # Flag to keep track whether this stream has initiated the request + self.request_sent = False + # Each time we send a data frame, we will decrease value by the amount send. self.remaining_content_length = self.content_length @@ -58,20 +68,30 @@ class Stream: # Flag to keep track whether the server has closed the stream self.stream_closed_server = False - # Data received frame by frame from the server is appended - # and passed to the response Deferred when completely received. - self._response_data = b"" - # The amount of data received that counts against the flow control # window self._response_flow_controlled_size = 0 - # Headers received after sending the request - self._response_headers = Headers({}) + # Private variable used to build the response + # this response is then converted to appropriate Response class + # passed to the response deferred callback + self._response = { + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. + 'body': b'', + + # Headers received after sending the request + 'headers': Headers({}) + } # TODO: Add canceller for the Deferred below self._deferred_response = Deferred() + def __str__(self): + return "Stream(id={})".format(self.stream_id) + + __repr__ = __str__ + def get_response(self): """Simply return a Deferred which fires when response from the asynchronous request is available @@ -104,6 +124,8 @@ class Stream: self._conn.send_headers(self.stream_id, headers, end_stream=False) self._write_to_transport() + self.request_sent = True + self.send_data() def send_data(self): @@ -112,7 +134,18 @@ class Stream: If the content length is 0 initially then we end the stream immediately and wait for response data. + + Warning: Only call this method when stream not closed from client side + and has initiated request already by sending HEADER frame. If not then + stream will be closed from client side with 499 response. + + TODO: Q. Should we instead raise ProtocolError here with a proper message? """ + if self.stream_closed_local or self.stream_closed_server: + raise StreamClosedError(self.stream_id) + elif not self.request_sent: + self.close() + return # TODO: # 1. Add test for sending very large data @@ -132,7 +165,8 @@ class Stream: bytes_to_send_size = min(window_size, self.remaining_content_length) # We now need to send a number of data frames. - while bytes_to_send_size: + data_frames_sent = 0 + while bytes_to_send_size > 0: chunk_size = min(bytes_to_send_size, max_frame_size) data_chunk_start_id = self.content_length - self.remaining_content_length @@ -140,16 +174,24 @@ class Stream: self._conn.send_data(self.stream_id, data_chunk, end_stream=False) + data_frames_sent += 1 bytes_to_send_size = bytes_to_send_size - chunk_size self.remaining_content_length = self.remaining_content_length - chunk_size - # End the stream if no more data has to be send - if self.remaining_content_length <= 0: - self.remaining_content_length = 0 + self.remaining_content_length = max(0, self.remaining_content_length) + LOGGER.debug("{} sending {}/{} data bytes ({} frames) to {}".format( + self, + self.content_length - self.remaining_content_length, self.content_length, + data_frames_sent, + self._metadata['ip_address']) + ) + # End the stream if no more data needs to be send + if self.remaining_content_length == 0: self.stream_closed_local = True self._conn.end_stream(self.stream_id) + # Write data to transport -- Empty the outstanding data self._write_to_transport() # Q. What about the rest of the data? @@ -163,11 +205,11 @@ class Stream: Arguments: delta -- Window change delta """ - if self.stream_closed_local is False: + if self.remaining_content_length > 0 and not self.stream_closed_server: self.send_data() def receive_data(self, data: bytes, flow_controlled_length: int): - self._response_data += data + self._response['body'] += data self._response_flow_controlled_size += flow_controlled_length # Acknowledge the data received @@ -178,47 +220,69 @@ class Stream: def receive_headers(self, headers): for name, value in headers: - self._response_headers[name] = value + self._response['headers'][name] = value - def reset(self): - """Received a RST_STREAM -- forcefully reset""" - # TODO: - # Q1. Do we need to send the request again? - # Q2. What response should we send now? - self.stream_closed_server = True - self._cb_close(self.stream_id) + def close(self, event=None): + """Based on the event sent we will handle each case. - def lost_connection(self): - # TODO: Same as self.reset - self.stream_closed_server = True - self._cb_close(self.stream_id) - - def end_stream(self): - """Stream is ended by the server hence no further + event: StreamEnded + Stream is ended by the server hence no further data or headers should be expected on this stream. - We will call the response deferred callback passing the response object + + event: StreamReset + Stream reset via RST_FRAME by the upstream hence forcefully close + this stream and send TODO: ? + + event: None + No event is launched -- Hence we will simply close this stream """ + # TODO: In case of abruptly stream close + # Q1. Do we need to send the request again? + # Q2. What response should we send now? assert self.stream_closed_server is False self.stream_closed_server = True + if not isinstance(event, StreamEnded): + # TODO + # Stream was abruptly ended here + # Partial - Content-Length header not provided + pass + self._fire_response_deferred() self._cb_close(self.stream_id) - def _fire_response_deferred(self): + def _fire_response_deferred(self, flags=None): + """Builds response from the self._response dict + and fires the response deferred callback with the + generated response instance""" # TODO: # 1. Set flags, certificate, ip_address in response # 2. Should we fire this in case of # 2.1 StreamReset in between when data is received partially # 2.2 Forcefully closed the stream + # 3. Update Client Side Status Codes here - # NOTE: Presently on fired with successful response - response = Response( + response_cls = responsetypes.from_args( + headers=self._response['headers'], url=self._request.url, - status=self._response_headers[":status"], - headers=self._response_headers, - body=self._response_data, - request=self._request + body=self._response['body'] ) + + # If there is :status in headers then + # HTTP Status Code: 499 - Client Closed Request + status = self._response['headers'].get(':status', '499') + + response = response_cls( + url=self._request.url, + status=status, + headers=self._response['headers'], + body=self._response['body'], + request=self._request, + flags=flags, + certificate=self._metadata['certificate'], + ip_address=self._metadata['ip_address'] + ) + self._deferred_response.callback(response) From c74ef660c7b28cd69825a4b9843230a383f702f4 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 21 Jun 2020 09:34:23 +0530 Subject: [PATCH 012/114] feat: handle response for different reasons - Add StreamCloseReason enum - Send response for different cases considering download_warnsize, download_maxsize, fail_on_data_loss, connection lost, etc. --- scrapy/core/http2/protocol.py | 15 ++- scrapy/core/http2/stream.py | 209 ++++++++++++++++++++++++---------- 2 files changed, 155 insertions(+), 69 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index d6134183a..188c14c15 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -11,7 +11,7 @@ from h2.events import ( ) from twisted.internet.protocol import connectionDone, Protocol -from scrapy.core.http2.stream import Stream +from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.http import Request LOGGER = logging.getLogger(__name__) @@ -71,7 +71,7 @@ class H2ClientProtocol(Protocol): stream_id=stream_id, request=request, connection=self.conn, - metadata=self._metadata, + conn_metadata=self._metadata, write_to_transport=self._write_to_transport, cb_close=self._stream_close_cb ) @@ -133,7 +133,7 @@ class H2ClientProtocol(Protocol): """ # Pop all streams which were pending and were not yet started for stream_id in list(self.streams): - self.streams[stream_id].close() + self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST) self.conn.close_connection() @@ -180,19 +180,18 @@ class H2ClientProtocol(Protocol): def stream_ended(self, event: StreamEnded): stream_id = event.stream_id - self.streams[stream_id].close() + self.streams[stream_id].close(StreamCloseReason.ENDED) def stream_reset(self, event: StreamReset): # TODO: event.stream_id was abruptly closed # Q. What should be the response? (Failure/Partial/???) - self.streams[event.stream_id].close(event) + self.streams[event.stream_id].close(StreamCloseReason.RESET) def window_updated(self, event: WindowUpdated): stream_id = event.stream_id if stream_id != 0: - self.streams[stream_id].receive_window_update(event.delta) + self.streams[stream_id].receive_window_update() else: # Send leftover data for all the streams for stream in self.streams.values(): - if stream.request_sent: - stream.send_data() + stream.receive_window_update() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index e8b4471d6..07a4428c8 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,11 +1,15 @@ import logging +from enum import IntFlag, auto +from io import BytesIO from typing import Dict from urllib.parse import urlparse from h2.connection import H2Connection -from h2.events import StreamEnded +from h2.errors import ErrorCodes from h2.exceptions import StreamClosedError -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, CancelledError +from twisted.python.failure import Failure +from twisted.web.client import ResponseFailed from scrapy.http import Request from scrapy.http.headers import Headers @@ -14,6 +18,20 @@ from scrapy.responsetypes import responsetypes LOGGER = logging.getLogger(__name__) +class StreamCloseReason(IntFlag): + # Received a StreamEnded event + ENDED = auto() + + # Received a StreamReset event -- ended abruptly + RESET = auto() + + # Transport connection was lost + CONNECTION_LOST = auto() + + # Expected response body size is more than allowed limit + MAXSIZE_EXCEEDED = auto() + + class Stream: """Represents a single HTTP/2 Stream. @@ -26,13 +44,16 @@ class Stream: """ def __init__( - self, - stream_id: int, - request: Request, - connection: H2Connection, - metadata: Dict, - write_to_transport, - cb_close + self, + stream_id: int, + request: Request, + connection: H2Connection, + conn_metadata: Dict, + write_to_transport, + cb_close, + download_maxsize=0, + download_warnsize=0, + fail_on_data_loss=True ): """ Arguments: @@ -40,7 +61,7 @@ class Stream: uniquely identified by a single integer request {Request} -- HTTP request connection {H2Connection} -- HTTP/2 connection this stream belongs to. - metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection + conn_metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection write_to_transport {callable} -- Method used to write & send data to the server This method should be used whenever some frame is to be sent to the server. cb_close {callable} -- Method called when this stream is closed @@ -49,16 +70,24 @@ class Stream: self.stream_id = stream_id self._request = request self._conn = connection - self._metadata = metadata + self._conn_metadata = conn_metadata self._write_to_transport = write_to_transport self._cb_close = cb_close - self._request_body = self._request.body - self.content_length = 0 if self._request_body is None else len(self._request_body) + self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) + self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) + self._fail_on_dataloss = self._request.meta.get('download_fail_on_dataloss', fail_on_data_loss) + + self.request_start_time = None + + self.content_length = 0 if self._request.body is None else len(self._request.body) # Flag to keep track whether this stream has initiated the request self.request_sent = False + # Flag to track whether we have logged about exceeding download warnsize + self._reached_warnsize = False + # Each time we send a data frame, we will decrease value by the amount send. self.remaining_content_length = self.content_length @@ -68,17 +97,17 @@ class Stream: # Flag to keep track whether the server has closed the stream self.stream_closed_server = False - # The amount of data received that counts against the flow control - # window - self._response_flow_controlled_size = 0 - # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback self._response = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. - 'body': b'', + 'body': BytesIO(), + + # The amount of data received that counts against the flow control + # window + 'flow_controlled_size': 0, # Headers received after sending the request 'headers': Headers({}) @@ -137,16 +166,8 @@ class Stream: Warning: Only call this method when stream not closed from client side and has initiated request already by sending HEADER frame. If not then - stream will be closed from client side with 499 response. - - TODO: Q. Should we instead raise ProtocolError here with a proper message? + stream will raise ProtocolError (raise by h2 state machine). """ - if self.stream_closed_local or self.stream_closed_server: - raise StreamClosedError(self.stream_id) - elif not self.request_sent: - self.close() - return - # TODO: # 1. Add test for sending very large data # 2. Add test for small data @@ -154,6 +175,9 @@ class Stream: # 3.1 Large number of request # 3.2 Small number of requests + if self.stream_closed_local: + raise StreamClosedError(self.stream_id) + # Firstly, check what the flow control window is for current stream. window_size = self._conn.local_flow_control_window(stream_id=self.stream_id) @@ -170,7 +194,7 @@ class Stream: chunk_size = min(bytes_to_send_size, max_frame_size) data_chunk_start_id = self.content_length - self.remaining_content_length - data_chunk = self._request_body[data_chunk_start_id:data_chunk_start_id + chunk_size] + data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size] self._conn.send_data(self.stream_id, data_chunk, end_stream=False) @@ -183,7 +207,7 @@ class Stream: self, self.content_length - self.remaining_content_length, self.content_length, data_frames_sent, - self._metadata['ip_address']) + self._conn_metadata['ip_address']) ) # End the stream if no more data needs to be send @@ -197,24 +221,40 @@ class Stream: # Q. What about the rest of the data? # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame - def receive_window_update(self, delta): + def receive_window_update(self): """Flow control window size was changed. Send data that earlier could not be sent as we were blocked behind the flow control. - - Arguments: - delta -- Window change delta """ - if self.remaining_content_length > 0 and not self.stream_closed_server: + if self.remaining_content_length and not self.stream_closed_server and self.request_sent: self.send_data() def receive_data(self, data: bytes, flow_controlled_length: int): - self._response['body'] += data - self._response_flow_controlled_size += flow_controlled_length + self._response['body'].write(data) + self._response['flow_controlled_size'] += flow_controlled_length + + if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize: + # Clear buffer earlier to avoid keeping data in memory for a long time + self._response['body'].truncate(0) + self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) + return + + if self._download_warnsize \ + and self._response['flow_controlled_size'] > self._download_warnsize \ + and not self._reached_warnsize: + self._reached_warnsize = True + warning_msg = ('Received more ({bytes}) bytes than download ', + 'warn size ({warnsize}) in request {request}') + warning_args = { + 'bytes': self._response['flow_controlled_size'], + 'warnsize': self._download_warnsize, + 'request': self._request + } + LOGGER.warning(warning_msg, warning_args) # Acknowledge the data received self._conn.acknowledge_received_data( - self._response_flow_controlled_size, + self._response['flow_controlled_size'], self.stream_id ) @@ -222,35 +262,83 @@ class Stream: for name, value in headers: self._response['headers'][name] = value - def close(self, event=None): - """Based on the event sent we will handle each case. + # Check if we exceed the allowed max data size which can be received + expected_size = int(self._response['headers'].get(b'Content-Length', -1)) + if self._download_maxsize and expected_size > self._download_maxsize: + self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) + return - event: StreamEnded - Stream is ended by the server hence no further - data or headers should be expected on this stream. - We will call the response deferred callback passing - the response object + if self._download_warnsize and expected_size > self._download_warnsize: + warning_msg = ("Expected response size ({size}) larger than ", + "download warn size ({warnsize}) in request {request}.") + warning_args = { + 'size': expected_size, 'warnsize': self._download_warnsize, + 'request': self._request + } + LOGGER.warning(warning_msg, warning_args) - event: StreamReset - Stream reset via RST_FRAME by the upstream hence forcefully close - this stream and send TODO: ? + def reset_stream(self, reason=StreamCloseReason.RESET): + """Close this stream by sending a RST_FRAME to the remote peer""" + # TODO: Q. REFUSED_STREAM or CANCEL ? + if self.stream_closed_local: + raise StreamClosedError(self.stream_id) - event: None - No event is launched -- Hence we will simply close this stream + self.stream_closed_local = True + self._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) + self._write_to_transport() + self.close(reason) + + def _is_data_lost(self) -> bool: + assert self.stream_closed_server + + expected_size = self._response['flow_controlled_size'] + received_body_size = int(self._response['headers'][b'Content-Length']) + + return expected_size != received_body_size + + def close(self, reason: StreamCloseReason): + """Based on the reason sent we will handle each case. """ # TODO: In case of abruptly stream close # Q1. Do we need to send the request again? # Q2. What response should we send now? - assert self.stream_closed_server is False + if self.stream_closed_server: + raise StreamClosedError(self.stream_id) + self.stream_closed_server = True - if not isinstance(event, StreamEnded): - # TODO - # Stream was abruptly ended here - # Partial - Content-Length header not provided - pass + flags = None + if b'Content-Length' not in self._response['headers']: + # Missing Content-Length - PotentialDataLoss + flags = ['partial'] + elif self._is_data_lost(): + if self._fail_on_dataloss: + self._deferred_response.errback(ResponseFailed([Failure()])) + self._cb_close(self.stream_id) + return + else: + flags = ['dataloss'] + + if reason is StreamCloseReason.ENDED: + self._fire_response_deferred(flags) + + elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST): + # Stream was abruptly ended here + self._deferred_response.errback(ResponseFailed([Failure()])) + + elif reason is StreamCloseReason.MAXSIZE_EXCEEDED: + expected_size = int(self._response['headers'].get(b'Content-Length', -1)) + error_msg = ("Cancelling download of {url}: expected response " + "size ({size}) larger than download max size ({maxsize}).") + error_args = { + 'url': self._request.url, + 'size': expected_size, + 'maxsize': self._download_maxsize + } + + LOGGER.error(error_msg, error_args) + self._deferred_response.errback(CancelledError(error_msg.format(**error_args))) - self._fire_response_deferred() self._cb_close(self.stream_id) def _fire_response_deferred(self, flags=None): @@ -258,7 +346,6 @@ class Stream: and fires the response deferred callback with the generated response instance""" # TODO: - # 1. Set flags, certificate, ip_address in response # 2. Should we fire this in case of # 2.1 StreamReset in between when data is received partially # 2.2 Forcefully closed the stream @@ -270,7 +357,7 @@ class Stream: body=self._response['body'] ) - # If there is :status in headers then + # If there is no :status in headers then # HTTP Status Code: 499 - Client Closed Request status = self._response['headers'].get(':status', '499') @@ -278,11 +365,11 @@ class Stream: url=self._request.url, status=status, headers=self._response['headers'], - body=self._response['body'], + body=self._response['body'].getvalue(), request=self._request, flags=flags, - certificate=self._metadata['certificate'], - ip_address=self._metadata['ip_address'] + certificate=self._conn_metadata['certificate'], + ip_address=self._conn_metadata['ip_address'] ) self._deferred_response.callback(response) From a97ac0adf86b67a16f09c41f618f736adb872503 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 24 Jun 2020 06:40:20 +0530 Subject: [PATCH 013/114] test: GET request for HTTP2Client using mockserver --- scrapy/core/http2/protocol.py | 4 +- scrapy/core/http2/stream.py | 10 +--- tests/test_http2_client_protocol.py | 71 ++++++++++------------------- 3 files changed, 30 insertions(+), 55 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 188c14c15..455d8777e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -9,6 +9,8 @@ from h2.events import ( DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) + +from twisted.internet.ssl import Certificate from twisted.internet.protocol import connectionDone, Protocol from scrapy.core.http2.stream import Stream, StreamCloseReason @@ -115,7 +117,7 @@ class H2ClientProtocol(Protocol): self.destination = self.transport.getPeer() LOGGER.info('Connection made to {}'.format(self.destination)) - self._metadata['certificate'] = self.transport.getPeerCertificate() + self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) self.conn.initiate_connection() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 07a4428c8..112ce5bcd 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -117,7 +117,7 @@ class Stream: self._deferred_response = Deferred() def __str__(self): - return "Stream(id={})".format(self.stream_id) + return "Stream(id={})".format(repr(self.stream_id)) __repr__ = __str__ @@ -299,9 +299,6 @@ class Stream: def close(self, reason: StreamCloseReason): """Based on the reason sent we will handle each case. """ - # TODO: In case of abruptly stream close - # Q1. Do we need to send the request again? - # Q2. What response should we send now? if self.stream_closed_server: raise StreamClosedError(self.stream_id) @@ -346,10 +343,7 @@ class Stream: and fires the response deferred callback with the generated response instance""" # TODO: - # 2. Should we fire this in case of - # 2.1 StreamReset in between when data is received partially - # 2.2 Forcefully closed the stream - # 3. Update Client Side Status Codes here + # 1. Update Client Side Status Codes here response_cls = responsetypes.from_args( headers=self._response['headers'], diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 7830f7028..a67575d3c 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,71 +1,50 @@ -import os -import shutil +from urllib.parse import urlparse -from twisted.internet import defer, reactor +from twisted.internet import reactor from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint -from twisted.internet.ssl import optionsForClientTLS -from twisted.protocols.policies import WrappingFactory -from twisted.python.filepath import FilePath +from twisted.internet.ssl import CertificateOptions from twisted.trial import unittest -from twisted.web import static, server from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.http import Request -from tests.mockserver import ssl_context_factory +from scrapy.http import Request, Response +from tests.mockserver import MockServer class Http2ClientProtocolTestCase(unittest.TestCase): scheme = 'https' - # only used for HTTPS tests - file_key = 'keys/localhost.key' - file_certificate = 'keys/localhost.crt' - def setUp(self): # Start server for testing - self.path_temp = self.mktemp() - os.mkdir(self.path_temp) - FilePath(self.path_temp).child('file').setContent(b"0123456789") - r = static.File(self.path_temp) + self.mockserver = MockServer() + self.mockserver.__enter__() - self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) - self.host = 'localhost' - if self.scheme is 'https': - self.port = reactor.listenSSL( - 0, self.wrapper, - ssl_context_factory(self.file_key, self.file_certificate), - interface=self.host - ) + if self.scheme == 'https': + self.url = urlparse(self.mockserver.https_address) else: - self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) - - self.port_number = self.port.getHost().port - - # Connect to the server using the custom HTTP2ClientProtocol - options = optionsForClientTLS( - hostname=self.host, - acceptableProtocols=[b'h2'] - ) + self.url = urlparse(self.mockserver.http_address) self.protocol = H2ClientProtocol() - connectProtocol( - endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options), - protocol=self.protocol - ) + # Connect to the server using the custom HTTP2ClientProtocol + options = CertificateOptions(acceptableProtocols=[b'h2']) + endpoint = SSL4ClientEndpoint(reactor, self.url.hostname, self.url.port, options) + connectProtocol(endpoint, self.protocol) def getURL(self, path): - return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path) + return "{}://{}:{}/{}".format(self.url.scheme, self.url.hostname, self.url.port, path) - @defer.inlineCallbacks def tearDown(self): - yield self.port.stopListening() - shutil.rmtree(self.path_temp) + self.mockserver.__exit__(None, None, None) def test_download(self): - request = Request(self.getURL('file')) + request = Request(self.getURL('')) + + def assert_response(response: Response): + self.assertEqual(response.body, b'Scrapy mock HTTP server\n') + self.assertEqual(response.status, 200) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + d = self.protocol.request(request) - d.addCallback(lambda response: response.body) - d.addCallback(self.assertEqual, b"0123456789") + d.addCallback(assert_response) return d From 69f6d038c0bc51a9de706890621d2ce183f79e09 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 24 Jun 2020 07:06:32 +0530 Subject: [PATCH 014/114] feat: TypedDict for Stream._response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - remove test_protocol.py as working testing environment is setup 🙂🙃 - Add typing_extensions as dependency to support TypedDict for python<3.8 --- scrapy/core/http2/protocol.py | 12 +----- scrapy/core/http2/stream.py | 11 ++++- scrapy/core/http2/test_protocol.py | 67 ------------------------------ setup.py | 4 +- 4 files changed, 15 insertions(+), 79 deletions(-) delete mode 100644 scrapy/core/http2/test_protocol.py diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 455d8777e..9b8ec6c77 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -9,9 +9,8 @@ from h2.events import ( DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) - -from twisted.internet.ssl import Certificate from twisted.internet.protocol import connectionDone, Protocol +from twisted.internet.ssl import Certificate from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.http import Request @@ -22,12 +21,7 @@ LOGGER = logging.getLogger(__name__) class H2ClientProtocol(Protocol): # TODO: # 1. Check for user-agent while testing - # 2. Add support for cookies - # 3. Handle priority updates (Not required) - # 4. Handle case when received events have StreamID = 0 (applied to H2Connection) - # 1 & 2: - # - Automatically handled by the Request middleware - # - request.headers will have 'Set-Cookie' value + # 2. Handle case when received events have StreamID = 0 (applied to H2Connection) def __init__(self): config = H2Configuration(client_side=True, header_encoding='utf-8') @@ -185,8 +179,6 @@ class H2ClientProtocol(Protocol): self.streams[stream_id].close(StreamCloseReason.ENDED) def stream_reset(self, event: StreamReset): - # TODO: event.stream_id was abruptly closed - # Q. What should be the response? (Failure/Partial/???) self.streams[event.stream_id].close(StreamCloseReason.RESET) def window_updated(self, event: WindowUpdated): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 112ce5bcd..023f4f4eb 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -10,11 +10,20 @@ from h2.exceptions import StreamClosedError from twisted.internet.defer import Deferred, CancelledError from twisted.python.failure import Failure from twisted.web.client import ResponseFailed +# for python < 3.8 -- typing.TypedDict is undefined +from typing_extensions import TypedDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes + +class _ResponseTypedDict(TypedDict): + body: BytesIO + flow_controlled_size: int + headers: Headers + + LOGGER = logging.getLogger(__name__) @@ -100,7 +109,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response = { + self._response: _ResponseTypedDict = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. 'body': BytesIO(), diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.py deleted file mode 100644 index c7782a518..000000000 --- a/scrapy/core/http2/test_protocol.py +++ /dev/null @@ -1,67 +0,0 @@ -# This is simple script to test - -import json - -from twisted.internet import reactor -from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint -from twisted.internet.ssl import optionsForClientTLS - -from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.http import Request, Response, JsonRequest - -try: - with open('data.json', 'r') as f: - JSON_DATA = json.load(f) -except: - JSON_DATA = { - "data": "To test for really large amount of data -- Add data.json with lots of data.", - "why": "To test whether correct data is sent :)" - } - -# Use nghttp2 for testing whether basic setup works - for small response -HTTPBIN_AUTHORITY = u'nghttp2.org' -HTTPBIN_REQUEST_URLS = 1 * [ - Request(url='https://nghttp2.org/httpbin/get', method='GET'), - Request(url='https://nghttp2.org/httpbin/post', method='POST'), - JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA), -] - -# Use POKE_API for testing large responses -POKE_API_AUTHORITY = u'pokeapi.co' -POKE_API_REQUESTS = 15 * [ - Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'), - Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'), - Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'), - Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'), # should give 404 -] - -AUTHORITY = POKE_API_AUTHORITY -REQUEST_URLS = POKE_API_REQUESTS - -options = optionsForClientTLS( - hostname=AUTHORITY, - acceptableProtocols=[b'h2'], -) - -protocol = H2ClientProtocol() - -count_responses = 1 - - -def print_response(response): - global count_responses - assert isinstance(response, Response) - print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body))) - count_responses = count_responses + 1 - - -for request in REQUEST_URLS: - d = protocol.request(request) - d.addCallback(print_response) - -connectProtocol( - SSL4ClientEndpoint(reactor, AUTHORITY, 443, options), - protocol -) - -reactor.run() diff --git a/setup.py b/setup.py index dafa5684a..d1470df5e 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=3.5', + python_requires='>=3.5.2', install_requires=[ 'Twisted>=17.9.0', 'Twisted[http2]>=17.9.0' @@ -80,6 +80,8 @@ setup( 'w3lib>=1.17.0', 'zope.interface>=4.1.3', 'protego>=0.1.15', + 'itemadapter>=0.1.0', + 'typing_extensions>=3.7' ], extras_require=extras_require, ) From 690dd7f38bfd245428bdc618dcca1fbc26284df1 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 28 Jun 2020 16:35:04 +0530 Subject: [PATCH 015/114] test: GET & POST request test for h2 client - Remove repeated dependency Twisted from setup.py - Test for both GET & POST when - Only 1 request - Large number (=20) of requests and - Small Data (10 KB) per request - Large Data (10 MB) per request - Test when request is cancelled by the client' BREAKING CHANGES Tests raises OpenSSL.SSL.Error when run using tox. However, all tests passes when ran using `python -m unittest`. --- scrapy/core/http2/protocol.py | 4 +- scrapy/core/http2/stream.py | 66 +++--- setup.py | 1 - tests/test_http2_client_protocol.py | 321 +++++++++++++++++++++++++--- 4 files changed, 331 insertions(+), 61 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 9b8ec6c77..7fb935f10 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -129,11 +129,11 @@ class H2ClientProtocol(Protocol): """ # Pop all streams which were pending and were not yet started for stream_id in list(self.streams): - self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST) + self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST, reason) self.conn.close_connection() - LOGGER.info("Connection lost with reason " + str(reason)) + LOGGER.warning("Connection lost with reason " + str(reason)) def _handle_events(self, events): """Private method which acts as a bridge between the events diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 023f4f4eb..a26a33918 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -19,8 +19,15 @@ from scrapy.responsetypes import responsetypes class _ResponseTypedDict(TypedDict): + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. body: BytesIO + + # The amount of data received that counts against the flow control + # window flow_controlled_size: int + + # Headers received after sending the request headers: Headers @@ -40,6 +47,9 @@ class StreamCloseReason(IntFlag): # Expected response body size is more than allowed limit MAXSIZE_EXCEEDED = auto() + # When the response deferred is cancelled + CANCELLED = auto() + class Stream: """Represents a single HTTP/2 Stream. @@ -110,20 +120,16 @@ class Stream: # this response is then converted to appropriate Response class # passed to the response deferred callback self._response: _ResponseTypedDict = { - # Data received frame by frame from the server is appended - # and passed to the response Deferred when completely received. 'body': BytesIO(), - - # The amount of data received that counts against the flow control - # window 'flow_controlled_size': 0, - - # Headers received after sending the request 'headers': Headers({}) } - # TODO: Add canceller for the Deferred below - self._deferred_response = Deferred() + def _cancel(_): + # Close this stream as gracefully as possible :) + self.reset_stream(StreamCloseReason.CANCELLED) + + self._deferred_response = Deferred(_cancel) def __str__(self): return "Stream(id={})".format(repr(self.stream_id)) @@ -177,13 +183,6 @@ class Stream: and has initiated request already by sending HEADER frame. If not then stream will raise ProtocolError (raise by h2 state machine). """ - # TODO: - # 1. Add test for sending very large data - # 2. Add test for small data - # 3. Both (1) and (2) should be tested for - # 3.1 Large number of request - # 3.2 Small number of requests - if self.stream_closed_local: raise StreamClosedError(self.stream_id) @@ -221,7 +220,6 @@ class Stream: # End the stream if no more data needs to be send if self.remaining_content_length == 0: - self.stream_closed_local = True self._conn.end_stream(self.stream_id) # Write data to transport -- Empty the outstanding data @@ -288,7 +286,6 @@ class Stream: def reset_stream(self, reason=StreamCloseReason.RESET): """Close this stream by sending a RST_FRAME to the remote peer""" - # TODO: Q. REFUSED_STREAM or CANCEL ? if self.stream_closed_local: raise StreamClosedError(self.stream_id) @@ -305,14 +302,16 @@ class Stream: return expected_size != received_body_size - def close(self, reason: StreamCloseReason): + def close(self, reason: StreamCloseReason, failure=None): """Based on the reason sent we will handle each case. """ if self.stream_closed_server: raise StreamClosedError(self.stream_id) + self._cb_close(self.stream_id) self.stream_closed_server = True + # Do nothing if the response deferred was cancelled flags = None if b'Content-Length' not in self._response['headers']: # Missing Content-Length - PotentialDataLoss @@ -320,7 +319,6 @@ class Stream: elif self._is_data_lost(): if self._fail_on_dataloss: self._deferred_response.errback(ResponseFailed([Failure()])) - self._cb_close(self.stream_id) return else: flags = ['dataloss'] @@ -328,10 +326,19 @@ class Stream: if reason is StreamCloseReason.ENDED: self._fire_response_deferred(flags) - elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST): - # Stream was abruptly ended here - self._deferred_response.errback(ResponseFailed([Failure()])) + # Stream was abruptly ended here + elif reason is StreamCloseReason.CANCELLED: + # Client has cancelled the request. Remove all the data + # received and fire the response deferred with no flags set + self._response['body'].truncate(0) + self._response['headers'].clear() + self._fire_response_deferred() + elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST): + if failure is None: + self._deferred_response.errback(ResponseFailed([Failure()])) + else: + self._deferred_response.errback(failure) elif reason is StreamCloseReason.MAXSIZE_EXCEEDED: expected_size = int(self._response['headers'].get(b'Content-Length', -1)) error_msg = ("Cancelling download of {url}: expected response " @@ -345,22 +352,21 @@ class Stream: LOGGER.error(error_msg, error_args) self._deferred_response.errback(CancelledError(error_msg.format(**error_args))) - self._cb_close(self.stream_id) - def _fire_response_deferred(self, flags=None): """Builds response from the self._response dict and fires the response deferred callback with the generated response instance""" - # TODO: - # 1. Update Client Side Status Codes here + # TODO: Update Client Side Status Codes here + body = self._response['body'].getvalue() response_cls = responsetypes.from_args( headers=self._response['headers'], url=self._request.url, - body=self._response['body'] + body=body ) - # If there is no :status in headers then + # If there is no :status in headers + # (happens when client called response_deferred.cancel()) # HTTP Status Code: 499 - Client Closed Request status = self._response['headers'].get(':status', '499') @@ -368,7 +374,7 @@ class Stream: url=self._request.url, status=status, headers=self._response['headers'], - body=self._response['body'].getvalue(), + body=body, request=self._request, flags=flags, certificate=self._conn_metadata['certificate'], diff --git a/setup.py b/setup.py index d1470df5e..575c74e7f 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,6 @@ setup( ], python_requires='>=3.5.2', install_requires=[ - 'Twisted>=17.9.0', 'Twisted[http2]>=17.9.0' 'cryptography>=2.0', 'cssselect>=0.9.1', diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index a67575d3c..0f3730c9e 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,50 +1,315 @@ -from urllib.parse import urlparse +# TODO: Add test cases for +# 1. No Content Length response header +# 2. Cancel Response Deferred +import json +import os +import random +import shutil +import string from twisted.internet import reactor -from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint -from twisted.internet.ssl import CertificateOptions -from twisted.trial import unittest +from twisted.internet.defer import inlineCallbacks, DeferredList +from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint +from twisted.internet.protocol import Factory +from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate +from twisted.trial.unittest import TestCase +from twisted.web.http import Request as TxRequest +from twisted.web.resource import Resource +from twisted.web.server import Site +from twisted.web.static import File from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.http import Request, Response -from tests.mockserver import MockServer +from scrapy.http import Request, Response, JsonRequest +from tests.mockserver import ssl_context_factory -class Http2ClientProtocolTestCase(unittest.TestCase): +def generate_random_string(size): + return ''.join(random.choices( + string.ascii_uppercase + string.digits, + k=size + )) + + +def make_html_body(val): + response = ''' +

Hello from HTTP2

+

{}

+'''.format(val) + return bytes(response, 'utf-8') + + +class Data: + SMALL_SIZE = 1024 * 10 # 10 KB + LARGE_SIZE = (1024 ** 2) * 10 # 10 MB + + STR_SMALL = generate_random_string(SMALL_SIZE) + STR_LARGE = generate_random_string(LARGE_SIZE) + + EXTRA_SMALL = generate_random_string(1024 * 15) + EXTRA_LARGE = generate_random_string((1024 ** 2) * 15) + + HTML_SMALL = make_html_body(STR_SMALL) + HTML_LARGE = make_html_body(STR_LARGE) + + JSON_SMALL = {'data': STR_SMALL} + JSON_LARGE = {'data': STR_LARGE} + + +class LeafResource(Resource): + isLeaf = True + + +class GetDataHtmlSmall(LeafResource): + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'text/html; charset=UTF-8') + return Data.HTML_SMALL + + +class GetDataHtmlLarge(LeafResource): + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'text/html; charset=UTF-8') + return Data.HTML_LARGE + + +class PostDataJsonMixin: + @staticmethod + def make_response(request: TxRequest, extra_data: str): + response = { + 'request-headers': {}, + 'request-body': json.loads(request.content.read()), + 'extra-data': extra_data + } + for k, v in request.requestHeaders.getAllRawHeaders(): + response['request-headers'][k.decode('utf-8')] = v[0].decode('utf-8') + + response_bytes = bytes(json.dumps(response), 'utf-8') + request.setHeader('Content-Type', 'application/json') + return response_bytes + + +class PostDataJsonSmall(LeafResource, PostDataJsonMixin): + def render_POST(self, request: TxRequest): + return self.make_response(request, Data.EXTRA_SMALL) + + +class PostDataJsonLarge(LeafResource, PostDataJsonMixin): + def render_POST(self, request: TxRequest): + return self.make_response(request, Data.EXTRA_LARGE) + + +def get_client_certificate(key_file, certificate_file): + with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: + pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) + + return PrivateCertificate.loadPEM(pem) + + +class Https2ClientProtocolTestCase(TestCase): scheme = 'https' + key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key') + certificate_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.crt') + def _init_resource(self): + self.temp_directory = self.mktemp() + os.mkdir(self.temp_directory) + r = File(self.temp_directory) + r.putChild(b'get-data-html-small', GetDataHtmlSmall()) + r.putChild(b'get-data-html-large', GetDataHtmlLarge()) + + r.putChild(b'post-data-json-small', PostDataJsonSmall()) + r.putChild(b'post-data-json-large', PostDataJsonLarge()) + return r + + @inlineCallbacks def setUp(self): + # Initialize resource tree + root = self._init_resource() + self.site = Site(root, timeout=None) + # Start server for testing - self.mockserver = MockServer() - self.mockserver.__enter__() - + self.hostname = u'localhost' if self.scheme == 'https': - self.url = urlparse(self.mockserver.https_address) + context_factory = ssl_context_factory(self.key_file, self.certificate_file) + server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) else: - self.url = urlparse(self.mockserver.http_address) + server_endpoint = TCP4ServerEndpoint(reactor, 0, interface=self.hostname) + self.server = yield server_endpoint.listen(self.site) + self.port_number = self.server.getHost().port - self.protocol = H2ClientProtocol() - - # Connect to the server using the custom HTTP2ClientProtocol - options = CertificateOptions(acceptableProtocols=[b'h2']) - endpoint = SSL4ClientEndpoint(reactor, self.url.hostname, self.url.port, options) - connectProtocol(endpoint, self.protocol) - - def getURL(self, path): - return "{}://{}:{}/{}".format(self.url.scheme, self.url.hostname, self.url.port, path) + # Connect H2 client with server + client_certificate = get_client_certificate(self.key_file, self.certificate_file) + client_options = optionsForClientTLS( + hostname=self.hostname, + trustRoot=client_certificate, + acceptableProtocols=[b'h2'] + ) + h2_client_factory = Factory.forProtocol(H2ClientProtocol) + client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) + self.client = yield client_endpoint.connect(h2_client_factory) + @inlineCallbacks def tearDown(self): - self.mockserver.__exit__(None, None, None) + yield self.client.transport.loseConnection() + yield self.client.transport.abortConnection() + yield self.server.stopListening() + shutil.rmtree(self.temp_directory) - def test_download(self): - request = Request(self.getURL('')) + def get_url(self, path): + """ + :param path: Should have / at the starting compulsorily if not empty + :return: Complete url + """ + assert len(path) > 0 and (path[0] == '/' or path[0] == '&') + return "{}://{}:{}{}".format(self.scheme, self.hostname, self.port_number, path) - def assert_response(response: Response): - self.assertEqual(response.body, b'Scrapy mock HTTP server\n') - self.assertEqual(response.status, 200) + @staticmethod + def _check_repeat(get_deferred, count): + d_list = [] + for _ in range(count): + d = get_deferred() + d_list.append(d) + + return DeferredList(d_list, fireOnOneErrback=True) + + def _check_GET( + self, + request: Request, + expected_body, + expected_status + ): + def check_response(response: Response): + self.assertEqual(response.status, expected_status) + self.assertEqual(response.body, expected_body) self.assertEqual(response.request, request) self.assertEqual(response.url, request.url) - d = self.protocol.request(request) + content_length = int(response.headers.get('Content-Length')) + self.assertEqual(len(response.body), content_length) + + d = self.client.request(request) + d.addCallback(check_response) + d.addErrback(self.fail) + return d + + def test_GET_small_body(self): + request = Request(self.get_url('/get-data-html-small')) + return self._check_GET(request, Data.HTML_SMALL, 200) + + def test_GET_large_body(self): + request = Request(self.get_url('/get-data-html-large')) + return self._check_GET(request, Data.HTML_LARGE, 200) + + def _check_GET_x20(self, *args, **kwargs): + def get_deferred(): + return self._check_GET(*args, **kwargs) + + return self._check_repeat(get_deferred, 20) + + def test_GET_small_body_x20(self): + return self._check_GET_x20( + Request(self.get_url('/get-data-html-small')), + Data.HTML_SMALL, + 200 + ) + + def test_GET_large_body_x20(self): + return self._check_GET_x20( + Request(self.get_url('/get-data-html-large')), + Data.HTML_LARGE, + 200 + ) + + def _check_POST_json( + self, + request: Request, + expected_request_body, + expected_extra_data, + expected_status: int + ): + d = self.client.request(request) + + def assert_response(response: Response): + self.assertEqual(response.status, expected_status) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + + content_length = int(response.headers.get('Content-Length')) + self.assertEqual(len(response.body), content_length) + + # Parse the body + body = json.loads(response.body.decode('utf-8')) + self.assertIn('request-body', body) + self.assertIn('extra-data', body) + self.assertIn('request-headers', body) + + request_body = body['request-body'] + self.assertEqual(request_body, expected_request_body) + + extra_data = body['extra-data'] + self.assertEqual(extra_data, expected_extra_data) + + # Check if headers were sent successfully + request_headers = body['request-headers'] + for k, v in request.headers.items(): + k_str = k.decode('utf-8') + self.assertIn(k_str, request_headers) + self.assertEqual(request_headers[k_str], v[0].decode('utf-8')) + d.addCallback(assert_response) return d + + def test_POST_small_json(self): + request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) + return self._check_POST_json( + request, + Data.JSON_SMALL, + Data.EXTRA_SMALL, + 200 + ) + + def test_POST_large_json(self): + request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) + return self._check_POST_json( + request, + Data.JSON_LARGE, + Data.EXTRA_LARGE, + 200 + ) + + def _check_POST_json_x20(self, *args, **kwargs): + def get_deferred(): + return self._check_POST_json(*args, **kwargs) + + return self._check_repeat(get_deferred, 20) + + def test_POST_small_json_x20(self): + request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) + return self._check_POST_json_x20( + request, + Data.JSON_SMALL, + Data.EXTRA_SMALL, + 200 + ) + + def test_POST_large_json_x20(self): + request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) + return self._check_POST_json_x20( + request, + Data.JSON_LARGE, + Data.EXTRA_LARGE, + 200 + ) + + def test_cancel_request(self): + request = Request(url=self.get_url('/get-data-html-large')) + + def assert_response(response: Response): + self.assertEqual(response.status, 499) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + + d = self.client.request(request) + d.addCallback(assert_response) + d.cancel() + + return d From 6387445ef519124f393a20657e66383340fb1677 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 28 Jun 2020 18:44:57 +0530 Subject: [PATCH 016/114] test(tox.ini): change Twisted -> Twisted[http2] --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 27d21ade2..ada211b3c 100644 --- a/tox.ini +++ b/tox.ini @@ -77,7 +77,7 @@ deps = pyOpenSSL==16.2.0 queuelib==1.4.2 service_identity==16.0.0 - Twisted==17.9.0 + Twisted[http2]==17.9.0 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt From 23906b6bee953d9bc5dd8042e785711b11840797 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 29 Jun 2020 18:21:05 +0530 Subject: [PATCH 017/114] refactor: move TypedDict types to types.py - rename LOGGER -> logger - remove self._write_to_transport from Stream class and handle all transport related activities inside HTTP2ClientProtocol class --- scrapy/core/http2/protocol.py | 69 +++++++----- scrapy/core/http2/stream.py | 206 +++++++++++++++++----------------- scrapy/core/http2/types.py | 30 +++++ setup.py | 2 +- 4 files changed, 174 insertions(+), 133 deletions(-) create mode 100644 scrapy/core/http2/types.py diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 7fb935f10..0b3e5d304 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -2,6 +2,7 @@ import ipaddress import itertools import logging from collections import deque +from typing import Union, Dict from h2.config import H2Configuration from h2.connection import H2Connection @@ -9,20 +10,18 @@ from h2.events import ( DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) +from h2.exceptions import ProtocolError from twisted.internet.protocol import connectionDone, Protocol from twisted.internet.ssl import Certificate from scrapy.core.http2.stream import Stream, StreamCloseReason +from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request -LOGGER = logging.getLogger(__name__) +logger = logging.getLogger(__name__) class H2ClientProtocol(Protocol): - # TODO: - # 1. Check for user-agent while testing - # 2. Handle case when received events have StreamID = 0 (applied to H2Connection) - def __init__(self): config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) @@ -37,7 +36,7 @@ class H2ClientProtocol(Protocol): self.stream_id_count = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs - self.streams = {} + self.streams: Dict[int, Stream] = {} # Boolean to keep track the connection is made # If requests are received before connection is made @@ -46,9 +45,11 @@ class H2ClientProtocol(Protocol): self.is_connection_made = False self._pending_request_stream_pool = deque() - # Some meta data of this connection - # initialized when connection is successfully made - self._metadata = { + # Save an instance of ProtocolError raised by hyper-h2 + # We pass this instance to the streams ResponseFailed() failure + self._protocol_error: Union[None, ProtocolError] = None + + self._metadata: H2ConnectionMetadataDict = { 'certificate': None, 'ip_address': None } @@ -68,7 +69,6 @@ class H2ClientProtocol(Protocol): request=request, connection=self.conn, conn_metadata=self._metadata, - write_to_transport=self._write_to_transport, cb_close=self._stream_close_cb ) @@ -89,10 +89,10 @@ class H2ClientProtocol(Protocol): data = self.conn.data_to_send() self.transport.write(data) - LOGGER.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address'])) + logger.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address'])) - def request(self, _request: Request): - stream = self._new_stream(_request) + def request(self, request: Request): + stream = self._new_stream(request) d = stream.get_response() # If connection is not yet established then add the @@ -109,7 +109,7 @@ class H2ClientProtocol(Protocol): sending some data now: we should open with the connection preamble. """ self.destination = self.transport.getPeer() - LOGGER.info('Connection made to {}'.format(self.destination)) + logger.info('Connection made to {}'.format(self.destination)) self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) @@ -119,21 +119,37 @@ class H2ClientProtocol(Protocol): self.is_connection_made = True def dataReceived(self, data): - events = self.conn.receive_data(data) - self._handle_events(events) - self._write_to_transport() + try: + events = self.conn.receive_data(data) + self._handle_events(events) + except ProtocolError as e: + # TODO: In case of InvalidBodyLengthError -- terminate only one stream + + # Save this error as ultimately the connection will be dropped + # internally by hyper-h2. Saved error will be passed to all the streams + # closed with the connection. + self._protocol_error = e + + # We lose the transport connection here + self.transport.loseConnection() + finally: + self._write_to_transport() def connectionLost(self, reason=connectionDone): """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ # Pop all streams which were pending and were not yet started - for stream_id in list(self.streams): - self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST, reason) + # NOTE: Stream.close() pops the element from the streams dictionary + # which raises `RuntimeError: dictionary changed size during iteration` + # Hence, we copy the streams into a list. + for stream in list(self.streams.values()): + stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error) self.conn.close_connection() - LOGGER.warning("Connection lost with reason " + str(reason)) + if not reason.check(connectionDone): + logger.warning("Connection lost with reason " + str(reason)) def _handle_events(self, events): """Private method which acts as a bridge between the events @@ -144,7 +160,7 @@ class H2ClientProtocol(Protocol): triggered by sending data """ for event in events: - LOGGER.debug(event) + logger.debug(event) if isinstance(event, DataReceived): self.data_received(event) elif isinstance(event, ResponseReceived): @@ -158,16 +174,14 @@ class H2ClientProtocol(Protocol): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) else: - LOGGER.info("Received unhandled event {}".format(event)) + logger.info("Received unhandled event {}".format(event)) # Event handler functions starts here def data_received(self, event: DataReceived): - stream_id = event.stream_id - self.streams[stream_id].receive_data(event.data, event.flow_controlled_length) + self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) def response_received(self, event: ResponseReceived): - stream_id = event.stream_id - self.streams[stream_id].receive_headers(event.headers) + self.streams[event.stream_id].receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged): # Send off all the pending requests @@ -175,8 +189,7 @@ class H2ClientProtocol(Protocol): self._send_pending_requests() def stream_ended(self, event: StreamEnded): - stream_id = event.stream_id - self.streams[stream_id].close(StreamCloseReason.ENDED) + self.streams[event.stream_id].close(StreamCloseReason.ENDED) def stream_reset(self, event: StreamReset): self.streams[event.stream_id].close(StreamCloseReason.RESET) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a26a33918..da0181d52 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,7 +1,7 @@ import logging -from enum import IntFlag, auto +from enum import Enum from io import BytesIO -from typing import Dict +from typing import Callable, List from urllib.parse import urlparse from h2.connection import H2Connection @@ -10,45 +10,31 @@ from h2.exceptions import StreamClosedError from twisted.internet.defer import Deferred, CancelledError from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -# for python < 3.8 -- typing.TypedDict is undefined -from typing_extensions import TypedDict +from scrapy.core.http2.types import H2ConnectionMetadataDict, H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes - -class _ResponseTypedDict(TypedDict): - # Data received frame by frame from the server is appended - # and passed to the response Deferred when completely received. - body: BytesIO - - # The amount of data received that counts against the flow control - # window - flow_controlled_size: int - - # Headers received after sending the request - headers: Headers +logger = logging.getLogger(__name__) -LOGGER = logging.getLogger(__name__) - - -class StreamCloseReason(IntFlag): +class StreamCloseReason(Enum): # Received a StreamEnded event - ENDED = auto() + ENDED = 1 # Received a StreamReset event -- ended abruptly - RESET = auto() + RESET = 2 # Transport connection was lost - CONNECTION_LOST = auto() + CONNECTION_LOST = 3 # Expected response body size is more than allowed limit - MAXSIZE_EXCEEDED = auto() + MAXSIZE_EXCEEDED = 4 - # When the response deferred is cancelled - CANCELLED = auto() + # When the response deferred is cancelled by the client + # (happens when client called response_deferred.cancel()) + CANCELLED = 5 class Stream: @@ -63,34 +49,30 @@ class Stream: """ def __init__( - self, - stream_id: int, - request: Request, - connection: H2Connection, - conn_metadata: Dict, - write_to_transport, - cb_close, - download_maxsize=0, - download_warnsize=0, - fail_on_data_loss=True + self, + stream_id: int, + request: Request, + connection: H2Connection, + conn_metadata: H2ConnectionMetadataDict, + cb_close: Callable[[int], None], + download_maxsize: int = 0, + download_warnsize: int = 0, + fail_on_data_loss: bool = True ): """ Arguments: - stream_id {int} -- For one HTTP/2 connection each stream is + stream_id -- For one HTTP/2 connection each stream is uniquely identified by a single integer - request {Request} -- HTTP request - connection {H2Connection} -- HTTP/2 connection this stream belongs to. - conn_metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection - write_to_transport {callable} -- Method used to write & send data to the server - This method should be used whenever some frame is to be sent to the server. - cb_close {callable} -- Method called when this stream is closed + request -- HTTP request + connection -- HTTP/2 connection this stream belongs to. + conn_metadata -- Reference to dictionary having metadata of HTTP/2 connection + cb_close -- Method called when this stream is closed to notify the TCP connection instance. """ self.stream_id = stream_id self._request = request self._conn = connection self._conn_metadata = conn_metadata - self._write_to_transport = write_to_transport self._cb_close = cb_close self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) @@ -119,7 +101,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: _ResponseTypedDict = { + self._response: H2ResponseDict = { 'body': BytesIO(), 'flow_controlled_size': 0, 'headers': Headers({}) @@ -136,6 +118,25 @@ class Stream: __repr__ = __str__ + @property + def _log_warnsize(self) -> bool: + """Checks if we have received data which exceeds the download warnsize + and whether we have not already logged about it. + + Returns: + True if both the above conditions hold true + False if any of the conditions is false + """ + content_length_header = int(self._response['headers'].get(b'Content-Length', -1)) + return ( + self._download_warnsize + and ( + self._response['flow_controlled_size'] > self._download_warnsize + or content_length_header > self._download_warnsize + ) + and not self._reached_warnsize + ) + def get_response(self): """Simply return a Deferred which fires when response from the asynchronous request is available @@ -166,10 +167,7 @@ class Stream: def initiate_request(self): headers = self._get_request_headers() self._conn.send_headers(self.stream_id, headers, end_stream=False) - self._write_to_transport() - self.request_sent = True - self.send_data() def send_data(self): @@ -211,20 +209,19 @@ class Stream: self.remaining_content_length = self.remaining_content_length - chunk_size self.remaining_content_length = max(0, self.remaining_content_length) - LOGGER.debug("{} sending {}/{} data bytes ({} frames) to {}".format( - self, - self.content_length - self.remaining_content_length, self.content_length, - data_frames_sent, - self._conn_metadata['ip_address']) + logger.debug( + "{stream} sending {received}/{expected} data bytes ({frames} frames) to {ip_address}".format( + stream=self, + received=self.content_length - self.remaining_content_length, + expected=self.content_length, + frames=data_frames_sent, + ip_address=self._conn_metadata['ip_address']) ) # End the stream if no more data needs to be send if self.remaining_content_length == 0: self._conn.end_stream(self.stream_id) - # Write data to transport -- Empty the outstanding data - self._write_to_transport() - # Q. What about the rest of the data? # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame @@ -240,24 +237,21 @@ class Stream: self._response['body'].write(data) self._response['flow_controlled_size'] += flow_controlled_length + # We check maxsize here in case the Content-Length header was not received if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize: - # Clear buffer earlier to avoid keeping data in memory for a long time - self._response['body'].truncate(0) self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) return - if self._download_warnsize \ - and self._response['flow_controlled_size'] > self._download_warnsize \ - and not self._reached_warnsize: + if self._log_warnsize: self._reached_warnsize = True - warning_msg = ('Received more ({bytes}) bytes than download ', - 'warn size ({warnsize}) in request {request}') + warning_msg = 'Received more ({bytes}) bytes than download ' \ + + 'warn size ({warnsize}) in request {request}' warning_args = { 'bytes': self._response['flow_controlled_size'], 'warnsize': self._download_warnsize, 'request': self._request } - LOGGER.warning(warning_msg, warning_args) + logger.warning(warning_msg.format(**warning_args)) # Acknowledge the data received self._conn.acknowledge_received_data( @@ -275,23 +269,27 @@ class Stream: self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) return - if self._download_warnsize and expected_size > self._download_warnsize: - warning_msg = ("Expected response size ({size}) larger than ", - "download warn size ({warnsize}) in request {request}.") + if self._log_warnsize: + self._reached_warnsize = True + warning_msg = 'Expected response size ({size}) larger than ' \ + + 'download warn size ({warnsize}) in request {request}' warning_args = { - 'size': expected_size, 'warnsize': self._download_warnsize, + 'size': expected_size, + 'warnsize': self._download_warnsize, 'request': self._request } - LOGGER.warning(warning_msg, warning_args) + logger.warning(warning_msg.format(**warning_args)) def reset_stream(self, reason=StreamCloseReason.RESET): """Close this stream by sending a RST_FRAME to the remote peer""" if self.stream_closed_local: raise StreamClosedError(self.stream_id) + # Clear buffer earlier to avoid keeping data in memory for a long time + self._response['body'].truncate(0) + self.stream_closed_local = True self._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) - self._write_to_transport() self.close(reason) def _is_data_lost(self) -> bool: @@ -302,8 +300,11 @@ class Stream: return expected_size != received_body_size - def close(self, reason: StreamCloseReason, failure=None): + def close(self, reason: StreamCloseReason, error: Exception = None): """Based on the reason sent we will handle each case. + + Arguments: + reason -- One if StreamCloseReason """ if self.stream_closed_server: raise StreamClosedError(self.stream_id) @@ -311,35 +312,16 @@ class Stream: self._cb_close(self.stream_id) self.stream_closed_server = True - # Do nothing if the response deferred was cancelled flags = None if b'Content-Length' not in self._response['headers']: - # Missing Content-Length - PotentialDataLoss + # Missing Content-Length - {twisted.web.http.PotentialDataLoss} flags = ['partial'] - elif self._is_data_lost(): - if self._fail_on_dataloss: - self._deferred_response.errback(ResponseFailed([Failure()])) - return - else: - flags = ['dataloss'] - if reason is StreamCloseReason.ENDED: - self._fire_response_deferred(flags) - - # Stream was abruptly ended here - elif reason is StreamCloseReason.CANCELLED: - # Client has cancelled the request. Remove all the data - # received and fire the response deferred with no flags set - self._response['body'].truncate(0) - self._response['headers'].clear() - self._fire_response_deferred() - - elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST): - if failure is None: - self._deferred_response.errback(ResponseFailed([Failure()])) - else: - self._deferred_response.errback(failure) - elif reason is StreamCloseReason.MAXSIZE_EXCEEDED: + # NOTE: Order of handling the events is important here + # As we immediately cancel the request when maxsize is exceeded while + # receiving DATA_FRAME's when we have received the headers (not + # having Content-Length) + if reason is StreamCloseReason.MAXSIZE_EXCEEDED: expected_size = int(self._response['headers'].get(b'Content-Length', -1)) error_msg = ("Cancelling download of {url}: expected response " "size ({size}) larger than download max size ({maxsize}).") @@ -349,14 +331,34 @@ class Stream: 'maxsize': self._download_maxsize } - LOGGER.error(error_msg, error_args) + logger.error(error_msg, error_args) self._deferred_response.errback(CancelledError(error_msg.format(**error_args))) - def _fire_response_deferred(self, flags=None): + elif reason is StreamCloseReason.ENDED: + self._fire_response_deferred(flags) + + # Stream was abruptly ended here + elif reason is StreamCloseReason.CANCELLED: + # Client has cancelled the request. Remove all the data + # received and fire the response deferred with no flags set + + # NOTE: The data is already flushed in Stream.reset_stream() called + # immediately when the stream needs to be cancelled + + # There maybe no :status in headers, we make + # HTTP Status Code: 499 - Client Closed Request + self._response['headers'][':status'] = '499' + self._fire_response_deferred() + + elif reason in (StreamCloseReason.RESET, StreamCloseReason.CONNECTION_LOST): + self._deferred_response.errback(ResponseFailed([ + error if error else Failure() + ])) + + def _fire_response_deferred(self, flags: List[str] = None): """Builds response from the self._response dict and fires the response deferred callback with the generated response instance""" - # TODO: Update Client Side Status Codes here body = self._response['body'].getvalue() response_cls = responsetypes.from_args( @@ -365,11 +367,7 @@ class Stream: body=body ) - # If there is no :status in headers - # (happens when client called response_deferred.cancel()) - # HTTP Status Code: 499 - Client Closed Request - status = self._response['headers'].get(':status', '499') - + status = self._response['headers'][':status'] response = response_cls( url=self._request.url, status=status, diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py new file mode 100644 index 000000000..f28bf9472 --- /dev/null +++ b/scrapy/core/http2/types.py @@ -0,0 +1,30 @@ +from io import BytesIO +from ipaddress import IPv4Address, IPv6Address +from typing import Union + +from twisted.internet.ssl import Certificate +# for python < 3.8 -- typing.TypedDict is undefined +from typing_extensions import TypedDict + +from scrapy.http.headers import Headers + + +class H2ConnectionMetadataDict(TypedDict): + """Some meta data of this connection + initialized when connection is successfully made + """ + certificate: Union[None, Certificate] + ip_address: Union[None, IPv4Address, IPv6Address] + + +class H2ResponseDict(TypedDict): + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. + body: BytesIO + + # The amount of data received that counts against the flow control + # window + flow_controlled_size: int + + # Headers received after sending the request + headers: Headers diff --git a/setup.py b/setup.py index 575c74e7f..8e50733e6 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ setup( ], python_requires='>=3.5.2', install_requires=[ - 'Twisted[http2]>=17.9.0' + 'Twisted[http2]>=17.9.0', 'cryptography>=2.0', 'cssselect>=0.9.1', 'lxml>=3.5.0', From 90a7007f8818dbc224dbe51b95f07199cb730204 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 29 Jun 2020 18:29:31 +0530 Subject: [PATCH 018/114] test: warnsize logs, no content header, dataloss --- tests/test_http2_client_protocol.py | 167 +++++++++++++++++++++++----- 1 file changed, 142 insertions(+), 25 deletions(-) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0f3730c9e..0a2719d23 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,26 +1,26 @@ -# TODO: Add test cases for -# 1. No Content Length response header -# 2. Cancel Response Deferred import json import os import random +import re import shutil import string +from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor -from twisted.internet.defer import inlineCallbacks, DeferredList +from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint from twisted.internet.protocol import Factory from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate +from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.http import Request as TxRequest -from twisted.web.resource import Resource -from twisted.web.server import Site +from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File from scrapy.core.http2.protocol import H2ClientProtocol from scrapy.http import Request, Response, JsonRequest -from tests.mockserver import ssl_context_factory +from scrapy.utils.python import to_bytes, to_unicode +from tests.mockserver import ssl_context_factory, LeafResource def generate_random_string(size): @@ -35,7 +35,7 @@ def make_html_body(val):

Hello from HTTP2

{}

'''.format(val) - return bytes(response, 'utf-8') + return to_bytes(response) class Data: @@ -54,9 +54,8 @@ class Data: JSON_SMALL = {'data': STR_SMALL} JSON_LARGE = {'data': STR_LARGE} - -class LeafResource(Resource): - isLeaf = True + DATALOSS = b'Dataloss Content' + NO_CONTENT_LENGTH = b'This response do not have any content-length header' class GetDataHtmlSmall(LeafResource): @@ -80,9 +79,9 @@ class PostDataJsonMixin: 'extra-data': extra_data } for k, v in request.requestHeaders.getAllRawHeaders(): - response['request-headers'][k.decode('utf-8')] = v[0].decode('utf-8') + response['request-headers'][to_unicode(k)] = to_unicode(v[0]) - response_bytes = bytes(json.dumps(response), 'utf-8') + response_bytes = to_bytes(json.dumps(response)) request.setHeader('Content-Type', 'application/json') return response_bytes @@ -97,6 +96,31 @@ class PostDataJsonLarge(LeafResource, PostDataJsonMixin): return self.make_response(request, Data.EXTRA_LARGE) +class Dataloss(LeafResource): + + def render_GET(self, request: TxRequest): + request.setHeader(b"Content-Length", b"1024") + self.deferRequest(request, 0, self._delayed_render, request) + return NOT_DONE_YET + + @staticmethod + def _delayed_render(request: TxRequest): + request.write(Data.DATALOSS) + request.finish() + + +class NoContentLengthHeader(LeafResource): + def render_GET(self, request: TxRequest): + request.requestHeaders.removeHeader('Content-Length') + self.deferRequest(request, 0, self._delayed_render, request) + return NOT_DONE_YET + + @staticmethod + def _delayed_render(request: TxRequest): + request.write(Data.NO_CONTENT_LENGTH) + request.finish() + + def get_client_certificate(key_file, certificate_file): with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) @@ -118,6 +142,9 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b'post-data-json-small', PostDataJsonSmall()) r.putChild(b'post-data-json-large', PostDataJsonLarge()) + + r.putChild(b'dataloss', Dataloss()) + r.putChild(b'no-content-length-header', NoContentLengthHeader()) return r @inlineCallbacks @@ -172,10 +199,10 @@ class Https2ClientProtocolTestCase(TestCase): return DeferredList(d_list, fireOnOneErrback=True) def _check_GET( - self, - request: Request, - expected_body, - expected_status + self, + request: Request, + expected_body, + expected_status ): def check_response(response: Response): self.assertEqual(response.status, expected_status) @@ -220,11 +247,11 @@ class Https2ClientProtocolTestCase(TestCase): ) def _check_POST_json( - self, - request: Request, - expected_request_body, - expected_extra_data, - expected_status: int + self, + request: Request, + expected_request_body, + expected_extra_data, + expected_status: int ): d = self.client.request(request) @@ -237,7 +264,7 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(len(response.body), content_length) # Parse the body - body = json.loads(response.body.decode('utf-8')) + body = json.loads(to_unicode(response.body)) self.assertIn('request-body', body) self.assertIn('extra-data', body) self.assertIn('request-headers', body) @@ -251,11 +278,12 @@ class Https2ClientProtocolTestCase(TestCase): # Check if headers were sent successfully request_headers = body['request-headers'] for k, v in request.headers.items(): - k_str = k.decode('utf-8') + k_str = to_unicode(k) self.assertIn(k_str, request_headers) - self.assertEqual(request_headers[k_str], v[0].decode('utf-8')) + self.assertEqual(request_headers[k_str], to_unicode(v[0])) d.addCallback(assert_response) + d.addErrback(self.fail) return d def test_POST_small_json(self): @@ -310,6 +338,95 @@ class Https2ClientProtocolTestCase(TestCase): d = self.client.request(request) d.addCallback(assert_response) + d.addErrback(self.fail) d.cancel() return d + + def test_download_maxsize_exceeded(self): + request = Request(url=self.get_url('/get-data-html-large'), meta={'download_maxsize': 1000}) + + def assert_cancelled_error(failure): + self.assertIsInstance(failure.value, CancelledError) + + d = self.client.request(request) + d.addCallback(self.fail) + d.addErrback(assert_cancelled_error) + return d + + # TODO: Test in multiple requests if one request fails due to dataloss + # remaining request do not fail (change expected behaviour) + # Can be done only when hyper-h2 don't terminate connection over + # InvalidBodyLengthError check + def test_received_dataloss_response(self): + """In case when value of Header Content-Length != len(Received Data) + ProtocolError is raised""" + request = Request(url=self.get_url('/dataloss')) + + def assert_failure(failure: Failure): + self.assertTrue(len(failure.value.reasons) > 0) + self.assertTrue(any( + isinstance(error, InvalidBodyLengthError) + for error in failure.value.reasons + )) + + d = self.client.request(request) + d.addCallback(self.fail) + d.addErrback(assert_failure) + return d + + def test_missing_content_length_header(self): + request = Request(url=self.get_url('/no-content-length-header')) + + def assert_content_length(response: Response): + self.assertEqual(response.status, 200) + self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + self.assertIn('partial', response.flags) + self.assertNotIn('Content-Length', response.headers) + + d = self.client.request(request) + d.addCallback(assert_content_length) + d.addErrback(self.fail) + return d + + @inlineCallbacks + def _check_log_warnsize( + self, + request, + warn_pattern, + expected_body + ): + with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm: + response = yield self.client.request(request) + self.assertEqual(response.status, 200) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, expected_body) + + # Check the warning is raised only once for this request + self.assertEqual(sum( + len(re.findall(warn_pattern, log)) + for log in cm.output + ), 1) + + @inlineCallbacks + def test_log_expected_warnsize(self): + request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000}) + warn_pattern = re.compile( + r'Expected response size \(\d*\) larger than ' + r'download warn size \(1000\) in request {}'.format(request) + ) + + yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) + + @inlineCallbacks + def test_log_received_warnsize(self): + request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10}) + warn_pattern = re.compile( + r'Received more \(\d*\) bytes than download ' + r'warn size \(10\) in request {}'.format(request) + ) + + yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) From 26ab3e4137ddee3c643ae63c0709529efc698433 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 30 Jun 2020 06:44:20 +0530 Subject: [PATCH 019/114] feat: FIFO policy to handle large no. of requests - add required test -- test by sending 1000 requests - increase test timeout to 180 seconds to account for tests taking long time --- scrapy/core/http2/protocol.py | 87 +++++++++++++++++++---------- scrapy/core/http2/stream.py | 26 ++++++++- tests/test_http2_client_protocol.py | 51 ++++++++++++++++- 3 files changed, 128 insertions(+), 36 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 0b3e5d304..4de80c05e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -38,13 +38,16 @@ class H2ClientProtocol(Protocol): # Streams are stored in a dictionary keyed off their stream IDs self.streams: Dict[int, Stream] = {} - # Boolean to keep track the connection is made - # If requests are received before connection is made - # we keep all requests in a pool and send them as the connection - # is made - self.is_connection_made = False + # If requests are received before connection is made we keep + # all requests in a pool and send them as the connection is made self._pending_request_stream_pool = deque() + # Counter to keep track of opened stream. This counter + # is used to make sure that not more than MAX_CONCURRENT_STREAMS + # streams are opened which leads to ProtocolError + # We use simple FIFO policy to handle pending requests + self._active_streams = 0 + # Save an instance of ProtocolError raised by hyper-h2 # We pass this instance to the streams ResponseFailed() failure self._protocol_error: Union[None, ProtocolError] = None @@ -54,10 +57,46 @@ class H2ClientProtocol(Protocol): 'ip_address': None } + @property + def is_connected(self): + """Boolean to keep track of the connection status. + This is used while initiating pending streams to make sure + that we initiate stream only during active HTTP/2 Connection + """ + return bool(self.transport.connected) + + @property + def allowed_max_concurrent_streams(self) -> int: + """We keep total two streams for client (sending data) and + server side (receiving data) for a single request. To be safe + we choose the minimum. Since this value can change in event + RemoteSettingsChanged we make variable a property. + """ + return min( + self.conn.local_settings.max_concurrent_streams, + self.conn.remote_settings.max_concurrent_streams + ) + + def _send_pending_requests(self): + """Initiate all pending requests from the deque following FIFO + We make sure that at any time {allowed_max_concurrent_streams} + streams are active. + """ + while ( + self._pending_request_stream_pool + and self._active_streams < self.allowed_max_concurrent_streams + and self.is_connected + ): + self._active_streams += 1 + stream = self._pending_request_stream_pool.popleft() + stream.initiate_request() + def _stream_close_cb(self, stream_id: int): """Called when stream is closed completely """ - self.streams.pop(stream_id, None) + self.streams.pop(stream_id) + self._active_streams -= 1 + self._send_pending_requests() def _new_stream(self, request: Request): """Instantiates a new Stream object @@ -75,13 +114,6 @@ class H2ClientProtocol(Protocol): self.streams[stream.stream_id] = stream return stream - def _send_pending_requests(self): - # TODO: handle MAX_CONCURRENT_STREAMS - # Initiate all pending requests - while self._pending_request_stream_pool: - stream = self._pending_request_stream_pool.popleft() - stream.initiate_request() - def _write_to_transport(self): """ Write data to the underlying transport connection from the HTTP2 connection instance if any @@ -89,19 +121,12 @@ class H2ClientProtocol(Protocol): data = self.conn.data_to_send() self.transport.write(data) - logger.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address'])) - def request(self, request: Request): stream = self._new_stream(request) d = stream.get_response() - # If connection is not yet established then add the - # stream to pool or initiate request - if self.is_connection_made: - stream.initiate_request() - else: - self._pending_request_stream_pool.append(stream) - + # Add the stream to the request pool + self._pending_request_stream_pool.append(stream) return d def connectionMade(self): @@ -116,7 +141,6 @@ class H2ClientProtocol(Protocol): self.conn.initiate_connection() self._write_to_transport() - self.is_connection_made = True def dataReceived(self, data): try: @@ -144,7 +168,10 @@ class H2ClientProtocol(Protocol): # which raises `RuntimeError: dictionary changed size during iteration` # Hence, we copy the streams into a list. for stream in list(self.streams.values()): - stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error) + if stream.request_sent: + stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error) + else: + stream.close(StreamCloseReason.INACTIVE) self.conn.close_connection() @@ -160,7 +187,6 @@ class H2ClientProtocol(Protocol): triggered by sending data """ for event in events: - logger.debug(event) if isinstance(event, DataReceived): self.data_received(event) elif isinstance(event, ResponseReceived): @@ -174,7 +200,7 @@ class H2ClientProtocol(Protocol): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) else: - logger.info("Received unhandled event {}".format(event)) + logger.debug("Received unhandled event {}".format(event)) # Event handler functions starts here def data_received(self, event: DataReceived): @@ -184,8 +210,8 @@ class H2ClientProtocol(Protocol): self.streams[event.stream_id].receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged): - # Send off all the pending requests - # as now we have established a proper HTTP/2 connection + # Send off all the pending requests as now we have + # established a proper HTTP/2 connection self._send_pending_requests() def stream_ended(self, event: StreamEnded): @@ -195,9 +221,8 @@ class H2ClientProtocol(Protocol): self.streams[event.stream_id].close(StreamCloseReason.RESET) def window_updated(self, event: WindowUpdated): - stream_id = event.stream_id - if stream_id != 0: - self.streams[stream_id].receive_window_update() + if event.stream_id != 0: + self.streams[event.stream_id].receive_window_update() else: # Send leftover data for all the streams for stream in self.streams.values(): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index da0181d52..19b1825e4 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -8,6 +8,7 @@ from h2.connection import H2Connection from h2.errors import ErrorCodes from h2.exceptions import StreamClosedError from twisted.internet.defer import Deferred, CancelledError +from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed @@ -19,6 +20,15 @@ from scrapy.responsetypes import responsetypes logger = logging.getLogger(__name__) +class InactiveStreamClosed(ConnectionClosed): + """Connection was closed without sending request headers + of the stream. This happens when a stream is waiting for other + streams to close and connection is lost.""" + + def __init__(self, request: Request): + self.request = request + + class StreamCloseReason(Enum): # Received a StreamEnded event ENDED = 1 @@ -32,10 +42,13 @@ class StreamCloseReason(Enum): # Expected response body size is more than allowed limit MAXSIZE_EXCEEDED = 4 - # When the response deferred is cancelled by the client + # Response deferred is cancelled by the client # (happens when client called response_deferred.cancel()) CANCELLED = 5 + # Connection lost and the stream was not initiated + INACTIVE = 6 + class Stream: """Represents a single HTTP/2 Stream. @@ -108,8 +121,12 @@ class Stream: } def _cancel(_): - # Close this stream as gracefully as possible :) - self.reset_stream(StreamCloseReason.CANCELLED) + # Close this stream as gracefully as possible + # Check if the stream has started + if self.request_sent: + self.reset_stream(StreamCloseReason.CANCELLED) + else: + self.close(StreamCloseReason.CANCELLED) self._deferred_response = Deferred(_cancel) @@ -355,6 +372,9 @@ class Stream: error if error else Failure() ])) + elif reason is StreamCloseReason.INACTIVE: + self._deferred_response.errback(InactiveStreamClosed(self._request)) + def _fire_response_deferred(self, flags: List[str] = None): """Builds response from the self._response dict and fires the response deferred callback with the diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0a2719d23..0cb32dda6 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -18,6 +18,7 @@ from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.core.http2.stream import InactiveStreamClosed from scrapy.http import Request, Response, JsonRequest from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource @@ -174,10 +175,14 @@ class Https2ClientProtocolTestCase(TestCase): client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) + # Increase the total time taken for each tests + self.timeout = 180 # default is 120 seconds + @inlineCallbacks def tearDown(self): - yield self.client.transport.loseConnection() - yield self.client.transport.abortConnection() + if self.client.is_connected: + yield self.client.transport.loseConnection() + yield self.client.transport.abortConnection() yield self.server.stopListening() shutil.rmtree(self.temp_directory) @@ -430,3 +435,45 @@ class Https2ClientProtocolTestCase(TestCase): ) yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) + + def test_max_concurrent_streams(self): + """Send 1000 requests to check if we can handle + very large number of request + """ + + def get_deferred(): + return self._check_GET( + Request(self.get_url('/get-data-html-small')), + Data.HTML_SMALL, + 200 + ) + + return self._check_repeat(get_deferred, 1000) + + def test_inactive_stream(self): + """Here we send 110 requests considering the MAX_CONCURRENT_STREAMS + by default is 100. After sending the first 100 requests we close the + connection.""" + d_list = [] + + def assert_inactive_stream(failure): + self.assertIsNotNone(failure.check(InactiveStreamClosed)) + + # Send 100 request (we do not check the result) + for _ in range(100): + d = self.client.request(Request(self.get_url('/get-data-html-small'))) + d.addBoth(lambda _: None) + d_list.append(d) + + # Now send 10 extra request and save the response deferred in a list + for _ in range(10): + d = self.client.request(Request(self.get_url('/get-data-html-small'))) + d.addCallback(self.fail) + d.addErrback(assert_inactive_stream) + d_list.append(d) + + # Close the connection now to fire all the extra 10 requests errback + # with InactiveStreamClosed + self.client.transport.abortConnection() + + return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) From 50dd9271b4566785430106cfa9384d51103f73d9 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 30 Jun 2020 07:17:48 +0530 Subject: [PATCH 020/114] fix: disable redundant logs - while testing the job exceeded the maximum log length and was terminated - reduce the number of requests from 20 to 10 --- scrapy/core/http2/stream.py | 8 -------- tests/test_http2_client_protocol.py | 27 ++++++++++++--------------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 19b1825e4..8d0c6d94d 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -226,14 +226,6 @@ class Stream: self.remaining_content_length = self.remaining_content_length - chunk_size self.remaining_content_length = max(0, self.remaining_content_length) - logger.debug( - "{stream} sending {received}/{expected} data bytes ({frames} frames) to {ip_address}".format( - stream=self, - received=self.content_length - self.remaining_content_length, - expected=self.content_length, - frames=data_frames_sent, - ip_address=self._conn_metadata['ip_address']) - ) # End the stream if no more data needs to be send if self.remaining_content_length == 0: diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0cb32dda6..79c129d11 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -175,9 +175,6 @@ class Https2ClientProtocolTestCase(TestCase): client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) - # Increase the total time taken for each tests - self.timeout = 180 # default is 120 seconds - @inlineCallbacks def tearDown(self): if self.client.is_connected: @@ -231,21 +228,21 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(self.get_url('/get-data-html-large')) return self._check_GET(request, Data.HTML_LARGE, 200) - def _check_GET_x20(self, *args, **kwargs): + def _check_GET_x10(self, *args, **kwargs): def get_deferred(): return self._check_GET(*args, **kwargs) - return self._check_repeat(get_deferred, 20) + return self._check_repeat(get_deferred, 10) - def test_GET_small_body_x20(self): - return self._check_GET_x20( + def test_GET_small_body_x10(self): + return self._check_GET_x10( Request(self.get_url('/get-data-html-small')), Data.HTML_SMALL, 200 ) - def test_GET_large_body_x20(self): - return self._check_GET_x20( + def test_GET_large_body_x10(self): + return self._check_GET_x10( Request(self.get_url('/get-data-html-large')), Data.HTML_LARGE, 200 @@ -309,24 +306,24 @@ class Https2ClientProtocolTestCase(TestCase): 200 ) - def _check_POST_json_x20(self, *args, **kwargs): + def _check_POST_json_x10(self, *args, **kwargs): def get_deferred(): return self._check_POST_json(*args, **kwargs) - return self._check_repeat(get_deferred, 20) + return self._check_repeat(get_deferred, 10) - def test_POST_small_json_x20(self): + def test_POST_small_json_x10(self): request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) - return self._check_POST_json_x20( + return self._check_POST_json_x10( request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200 ) - def test_POST_large_json_x20(self): + def test_POST_large_json_x10(self): request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) - return self._check_POST_json_x20( + return self._check_POST_json_x10( request, Data.JSON_LARGE, Data.EXTRA_LARGE, From 7b1ad995a4996babf9a019815dd7256a1cbfa044 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 1 Jul 2020 10:45:36 +0530 Subject: [PATCH 021/114] test: query params, certificate & ip_address - refactor from str.format() to f-strings --- scrapy/core/http2/protocol.py | 12 ++-- scrapy/core/http2/stream.py | 46 +++++--------- scrapy/core/http2/types.py | 6 +- setup.py | 7 ++- tests/test_http2_client_protocol.py | 96 +++++++++++++++++++++++------ 5 files changed, 108 insertions(+), 59 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 4de80c05e..3438c99f0 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -122,6 +122,9 @@ class H2ClientProtocol(Protocol): self.transport.write(data) def request(self, request: Request): + if not isinstance(request, Request): + raise TypeError(f'Expected type scrapy.http.Request but received {request.__class__.__name__}') + stream = self._new_stream(request) d = stream.get_response() @@ -134,9 +137,7 @@ class H2ClientProtocol(Protocol): sending some data now: we should open with the connection preamble. """ self.destination = self.transport.getPeer() - logger.info('Connection made to {}'.format(self.destination)) - - self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) + logger.info(f'Connection made to {self.destination}') self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) self.conn.initiate_connection() @@ -200,7 +201,7 @@ class H2ClientProtocol(Protocol): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) else: - logger.debug("Received unhandled event {}".format(event)) + logger.debug(f'Received unhandled event {event}') # Event handler functions starts here def data_received(self, event: DataReceived): @@ -214,6 +215,9 @@ class H2ClientProtocol(Protocol): # established a proper HTTP/2 connection self._send_pending_requests() + # Update certificate when our HTTP/2 connection is established + self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) + def stream_ended(self, event: StreamEnded): self.streams[event.stream_id].close(StreamCloseReason.ENDED) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 8d0c6d94d..f4a90a753 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -131,7 +131,7 @@ class Stream: self._deferred_response = Deferred(_cancel) def __str__(self): - return "Stream(id={})".format(repr(self.stream_id)) + return f'Stream(id={self.stream_id})' __repr__ = __str__ @@ -167,13 +167,15 @@ class Stream: url = urlparse(self._request.url) # Make sure pseudo-headers comes before all the other headers + path = url.path + if url.query: + path += '?' + url.query + headers = [ (':method', self._request.method), (':authority', url.netloc), - - # TODO: Check if scheme can be 'http' for HTTP/2 ? (':scheme', 'https'), - (':path', url.path), + (':path', path), ] for name, value in self._request.headers.items(): @@ -253,14 +255,9 @@ class Stream: if self._log_warnsize: self._reached_warnsize = True - warning_msg = 'Received more ({bytes}) bytes than download ' \ - + 'warn size ({warnsize}) in request {request}' - warning_args = { - 'bytes': self._response['flow_controlled_size'], - 'warnsize': self._download_warnsize, - 'request': self._request - } - logger.warning(warning_msg.format(**warning_args)) + warning_msg = f"Received more ({self._response['flow_controlled_size']}) bytes than download " \ + + f'warn size ({self._download_warnsize}) in request {self._request}' + logger.warning(warning_msg) # Acknowledge the data received self._conn.acknowledge_received_data( @@ -280,14 +277,9 @@ class Stream: if self._log_warnsize: self._reached_warnsize = True - warning_msg = 'Expected response size ({size}) larger than ' \ - + 'download warn size ({warnsize}) in request {request}' - warning_args = { - 'size': expected_size, - 'warnsize': self._download_warnsize, - 'request': self._request - } - logger.warning(warning_msg.format(**warning_args)) + warning_msg = f'Expected response size ({expected_size}) larger than ' \ + + f'download warn size ({self._download_warnsize}) in request {self._request}' + logger.warning(warning_msg) def reset_stream(self, reason=StreamCloseReason.RESET): """Close this stream by sending a RST_FRAME to the remote peer""" @@ -332,16 +324,10 @@ class Stream: # having Content-Length) if reason is StreamCloseReason.MAXSIZE_EXCEEDED: expected_size = int(self._response['headers'].get(b'Content-Length', -1)) - error_msg = ("Cancelling download of {url}: expected response " - "size ({size}) larger than download max size ({maxsize}).") - error_args = { - 'url': self._request.url, - 'size': expected_size, - 'maxsize': self._download_maxsize - } - - logger.error(error_msg, error_args) - self._deferred_response.errback(CancelledError(error_msg.format(**error_args))) + error_msg = f'Cancelling download of {self._request.url}: expected response ' \ + f'size ({expected_size}) larger than download max size ({self._download_maxsize}).' + logger.error(error_msg) + self._deferred_response.errback(CancelledError(error_msg)) elif reason is StreamCloseReason.ENDED: self._fire_response_deferred(flags) diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py index f28bf9472..c0961cd3a 100644 --- a/scrapy/core/http2/types.py +++ b/scrapy/core/http2/types.py @@ -1,6 +1,6 @@ from io import BytesIO from ipaddress import IPv4Address, IPv6Address -from typing import Union +from typing import Union, Optional from twisted.internet.ssl import Certificate # for python < 3.8 -- typing.TypedDict is undefined @@ -13,8 +13,8 @@ class H2ConnectionMetadataDict(TypedDict): """Some meta data of this connection initialized when connection is successfully made """ - certificate: Union[None, Certificate] - ip_address: Union[None, IPv4Address, IPv6Address] + certificate: Optional[Certificate] + ip_address: Optional[Union[IPv4Address, IPv6Address]] class H2ResponseDict(TypedDict): diff --git a/setup.py b/setup.py index 8e50733e6..47c5906e4 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ from os.path import dirname, join - from pkg_resources import parse_version from setuptools import setup, find_packages, __version__ as setuptools_version + with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: version = f.read().decode('ascii').strip() @@ -25,11 +25,12 @@ if has_environment_marker_platform_impl_support(): 'PyPyDispatcher>=2.1.0', ] + setup( name='Scrapy', version=version, url='https://scrapy.org', - project_urls={ + project_urls = { 'Documentation': 'https://docs.scrapy.org/', 'Source': 'https://github.com/scrapy/scrapy', 'Tracker': 'https://github.com/scrapy/scrapy/issues', @@ -83,4 +84,4 @@ setup( 'typing_extensions>=3.7' ], extras_require=extras_require, -) +) \ No newline at end of file diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 79c129d11..c6a9bd5fb 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -4,13 +4,15 @@ import random import re import shutil import string +from ipaddress import IPv4Address +from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint from twisted.internet.protocol import Factory -from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate +from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.http import Request as TxRequest @@ -21,7 +23,7 @@ from scrapy.core.http2.protocol import H2ClientProtocol from scrapy.core.http2.stream import InactiveStreamClosed from scrapy.http import Request, Response, JsonRequest from scrapy.utils.python import to_bytes, to_unicode -from tests.mockserver import ssl_context_factory, LeafResource +from tests.mockserver import ssl_context_factory, LeafResource, Status def generate_random_string(size): @@ -32,10 +34,10 @@ def generate_random_string(size): def make_html_body(val): - response = ''' + response = f'''

Hello from HTTP2

-

{}

-'''.format(val) +

{val}

+''' return to_bytes(response) @@ -122,6 +124,17 @@ class NoContentLengthHeader(LeafResource): request.finish() +class QueryParams(LeafResource): + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'application/json') + + query_params = {} + for k, v in request.args.items(): + query_params[to_unicode(k)] = to_unicode(v[0]) + + return to_bytes(json.dumps(query_params)) + + def get_client_certificate(key_file, certificate_file): with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) @@ -146,6 +159,8 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b'dataloss', Dataloss()) r.putChild(b'no-content-length-header', NoContentLengthHeader()) + r.putChild(b'status', Status()) + r.putChild(b'query-params', QueryParams()) return r @inlineCallbacks @@ -189,7 +204,7 @@ class Https2ClientProtocolTestCase(TestCase): :return: Complete url """ assert len(path) > 0 and (path[0] == '/' or path[0] == '&') - return "{}://{}:{}{}".format(self.scheme, self.hostname, self.port_number, path) + return f'{self.scheme}://{self.hostname}:{self.port_number}{path}' @staticmethod def _check_repeat(get_deferred, count): @@ -210,7 +225,6 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, expected_status) self.assertEqual(response.body, expected_body) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) content_length = int(response.headers.get('Content-Length')) self.assertEqual(len(response.body), content_length) @@ -260,7 +274,6 @@ class Https2ClientProtocolTestCase(TestCase): def assert_response(response: Response): self.assertEqual(response.status, expected_status) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) content_length = int(response.headers.get('Content-Length')) self.assertEqual(len(response.body), content_length) @@ -336,7 +349,6 @@ class Https2ClientProtocolTestCase(TestCase): def assert_response(response: Response): self.assertEqual(response.status, 499) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) d = self.client.request(request) d.addCallback(assert_response) @@ -356,10 +368,6 @@ class Https2ClientProtocolTestCase(TestCase): d.addErrback(assert_cancelled_error) return d - # TODO: Test in multiple requests if one request fails due to dataloss - # remaining request do not fail (change expected behaviour) - # Can be done only when hyper-h2 don't terminate connection over - # InvalidBodyLengthError check def test_received_dataloss_response(self): """In case when value of Header Content-Length != len(Received Data) ProtocolError is raised""" @@ -384,7 +392,6 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, 200) self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) self.assertIn('partial', response.flags) self.assertNotIn('Content-Length', response.headers) @@ -404,7 +411,6 @@ class Https2ClientProtocolTestCase(TestCase): response = yield self.client.request(request) self.assertEqual(response.status, 200) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) self.assertEqual(response.body, expected_body) # Check the warning is raised only once for this request @@ -417,8 +423,8 @@ class Https2ClientProtocolTestCase(TestCase): def test_log_expected_warnsize(self): request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000}) warn_pattern = re.compile( - r'Expected response size \(\d*\) larger than ' - r'download warn size \(1000\) in request {}'.format(request) + rf'Expected response size \(\d*\) larger than ' + rf'download warn size \(1000\) in request {request}' ) yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) @@ -427,8 +433,8 @@ class Https2ClientProtocolTestCase(TestCase): def test_log_received_warnsize(self): request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10}) warn_pattern = re.compile( - r'Received more \(\d*\) bytes than download ' - r'warn size \(10\) in request {}'.format(request) + rf'Received more \(\d*\) bytes than download ' + rf'warn size \(10\) in request {request}' ) yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) @@ -474,3 +480,55 @@ class Https2ClientProtocolTestCase(TestCase): self.client.transport.abortConnection() return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) + + def test_invalid_request_type(self): + with self.assertRaises(TypeError): + self.client.request('https://InvalidDataTypePassed.com') + + def test_query_parameters(self): + params = { + 'a': generate_random_string(20), + 'b': generate_random_string(20), + 'c': generate_random_string(20), + 'd': generate_random_string(20) + } + request = Request(self.get_url(f'/query-params?{urlencode(params)}')) + + def assert_query_params(response: Response): + data = json.loads(to_unicode(response.body)) + self.assertEqual(data, params) + + d = self.client.request(request) + d.addCallback(assert_query_params) + d.addErrback(self.fail) + + return d + + def test_status_codes(self): + def assert_response_status(response: Response, expected_status: int): + self.assertEqual(response.status, expected_status) + + d_list = [] + for status in [200, 404]: + request = Request(self.get_url(f'/status?n={status}')) + d = self.client.request(request) + d.addCallback(assert_response_status, status) + d.addErrback(self.fail) + d_list.append(d) + + return DeferredList(d_list, fireOnOneErrback=True) + + def test_response_has_correct_certificate_ip_address(self): + request = Request(self.get_url('/status?n=200')) + + def assert_metadata(response: Response): + self.assertEqual(response.request, request) + self.assertIsInstance(response.certificate, Certificate) + self.assertIsInstance(response.ip_address, IPv4Address) + self.assertEqual(str(response.ip_address), '127.0.0.1') + + d = self.client.request(request) + d.addCallback(assert_metadata) + d.addErrback(self.fail) + + return d From c361fe0d3b80ff8a9f88adc05e730d5e469db225 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 1 Jul 2020 18:14:44 +0530 Subject: [PATCH 022/114] feat: check for invalid hostname - Initiating requests having hostname or (ip_address, port) different from the peer to which HTTP/2 connection is made can lead to closing the whole connection and close out all the pending streams. - This change aims to fix that problem - Add required tests - Save hostname & port in H2ConnectionMetadataDict --- scrapy/core/http2/protocol.py | 26 ++++++------ scrapy/core/http2/stream.py | 50 +++++++++++++++++++--- scrapy/core/http2/types.py | 11 +++++ tests/test_http2_client_protocol.py | 64 +++++++++++++++++++++++------ 4 files changed, 118 insertions(+), 33 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 3438c99f0..5de516482 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -2,7 +2,7 @@ import ipaddress import itertools import logging from collections import deque -from typing import Union, Dict +from typing import Dict, Optional from h2.config import H2Configuration from h2.connection import H2Connection @@ -26,10 +26,6 @@ class H2ClientProtocol(Protocol): config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) - # Address of the server we are connected to - # these are updated when connection is successfully made - self.destination = None - # ID of the next request stream # Following the convention made by hyper-h2 each client ID # will be odd. @@ -50,11 +46,13 @@ class H2ClientProtocol(Protocol): # Save an instance of ProtocolError raised by hyper-h2 # We pass this instance to the streams ResponseFailed() failure - self._protocol_error: Union[None, ProtocolError] = None + self._protocol_error: Optional[ProtocolError] = None self._metadata: H2ConnectionMetadataDict = { 'certificate': None, - 'ip_address': None + 'ip_address': None, + 'hostname': None, + 'port': None } @property @@ -123,7 +121,7 @@ class H2ClientProtocol(Protocol): def request(self, request: Request): if not isinstance(request, Request): - raise TypeError(f'Expected type scrapy.http.Request but received {request.__class__.__name__}') + raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__name__}') stream = self._new_stream(request) d = stream.get_response() @@ -136,9 +134,11 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ - self.destination = self.transport.getPeer() - logger.info(f'Connection made to {self.destination}') - self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) + destination = self.transport.getPeer() + logger.debug('Connection made to {}'.format(destination)) + self._metadata['ip_address'] = ipaddress.ip_address(destination.host) + self._metadata['port'] = destination.port + self._metadata['hostname'] = self.transport.transport.addr[0] self.conn.initiate_connection() self._write_to_transport() @@ -148,8 +148,6 @@ class H2ClientProtocol(Protocol): events = self.conn.receive_data(data) self._handle_events(events) except ProtocolError as e: - # TODO: In case of InvalidBodyLengthError -- terminate only one stream - # Save this error as ultimately the connection will be dropped # internally by hyper-h2. Saved error will be passed to all the streams # closed with the connection. @@ -201,7 +199,7 @@ class H2ClientProtocol(Protocol): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) else: - logger.debug(f'Received unhandled event {event}') + logger.debug('Received unhandled event {}'.format(event)) # Event handler functions starts here def data_received(self, event: DataReceived): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index f4a90a753..cc751b682 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -29,6 +29,17 @@ class InactiveStreamClosed(ConnectionClosed): self.request = request +class InvalidHostname(Exception): + + def __init__(self, request: Request, expected_hostname, expected_netloc): + self.request = request + self.expected_hostname = expected_hostname + self.expected_netloc = expected_netloc + + def __str__(self): + return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}' + + class StreamCloseReason(Enum): # Received a StreamEnded event ENDED = 1 @@ -49,6 +60,10 @@ class StreamCloseReason(Enum): # Connection lost and the stream was not initiated INACTIVE = 6 + # The hostname of the request is not same as of connected peer hostname + # As a result sending this request will the end the connection + INVALID_HOSTNAME = 7 + class Stream: """Represents a single HTTP/2 Stream. @@ -163,6 +178,15 @@ class Stream: """ return self._deferred_response + def check_request_url(self) -> bool: + # Make sure that we are sending the request to the correct URL + url = urlparse(self._request.url) + return ( + url.netloc == self._conn_metadata['hostname'] + or url.netloc == f'{self._conn_metadata["hostname"]}:{self._conn_metadata["port"]}' + or url.netloc == f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}' + ) + def _get_request_headers(self): url = urlparse(self._request.url) @@ -184,10 +208,15 @@ class Stream: return headers def initiate_request(self): - headers = self._get_request_headers() - self._conn.send_headers(self.stream_id, headers, end_stream=False) - self.request_sent = True - self.send_data() + if self.check_request_url(): + headers = self._get_request_headers() + self._conn.send_headers(self.stream_id, headers, end_stream=False) + self.request_sent = True + self.send_data() + else: + # Close this stream calling the response errback + # Note that we have not sent any headers + self.close(StreamCloseReason.INVALID_HOSTNAME) def send_data(self): """Called immediately after the headers are sent. Here we send all the @@ -310,6 +339,9 @@ class Stream: if self.stream_closed_server: raise StreamClosedError(self.stream_id) + if not isinstance(reason, StreamCloseReason): + raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__name__}') + self._cb_close(self.stream_id) self.stream_closed_server = True @@ -353,6 +385,13 @@ class Stream: elif reason is StreamCloseReason.INACTIVE: self._deferred_response.errback(InactiveStreamClosed(self._request)) + elif reason is StreamCloseReason.INVALID_HOSTNAME: + self._deferred_response.errback(InvalidHostname( + self._request, + self._conn_metadata['hostname'], + f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}' + )) + def _fire_response_deferred(self, flags: List[str] = None): """Builds response from the self._response dict and fires the response deferred callback with the @@ -365,10 +404,9 @@ class Stream: body=body ) - status = self._response['headers'][':status'] response = response_cls( url=self._request.url, - status=status, + status=self._response['headers'][':status'], headers=self._response['headers'], body=body, request=self._request, diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py index c0961cd3a..dd7b1187b 100644 --- a/scrapy/core/http2/types.py +++ b/scrapy/core/http2/types.py @@ -14,8 +14,19 @@ class H2ConnectionMetadataDict(TypedDict): initialized when connection is successfully made """ certificate: Optional[Certificate] + + # Address of the server we are connected to which + # is updated when HTTP/2 connection is made successfully ip_address: Optional[Union[IPv4Address, IPv6Address]] + # Name of the peer HTTP/2 connection is established + hostname: Optional[str] + + port: Optional[int] + + # Both ip_address and hostname are used by the Stream before + # initiating the request to verify that the base address + class H2ResponseDict(TypedDict): # Data received frame by frame from the server is appended diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index c6a9bd5fb..8f9fbe6fd 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -10,7 +10,7 @@ from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError -from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint +from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.protocol import Factory from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure @@ -20,7 +20,7 @@ from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.core.http2.stream import InactiveStreamClosed +from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -135,7 +135,7 @@ class QueryParams(LeafResource): return to_bytes(json.dumps(query_params)) -def get_client_certificate(key_file, certificate_file): +def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) @@ -171,19 +171,16 @@ class Https2ClientProtocolTestCase(TestCase): # Start server for testing self.hostname = u'localhost' - if self.scheme == 'https': - context_factory = ssl_context_factory(self.key_file, self.certificate_file) - server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) - else: - server_endpoint = TCP4ServerEndpoint(reactor, 0, interface=self.hostname) + context_factory = ssl_context_factory(self.key_file, self.certificate_file) + server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) self.server = yield server_endpoint.listen(self.site) self.port_number = self.server.getHost().port # Connect H2 client with server - client_certificate = get_client_certificate(self.key_file, self.certificate_file) + self.client_certificate = get_client_certificate(self.key_file, self.certificate_file) client_options = optionsForClientTLS( hostname=self.hostname, - trustRoot=client_certificate, + trustRoot=self.client_certificate, acceptableProtocols=[b'h2'] ) h2_client_factory = Factory.forProtocol(H2ClientProtocol) @@ -440,8 +437,8 @@ class Https2ClientProtocolTestCase(TestCase): yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) def test_max_concurrent_streams(self): - """Send 1000 requests to check if we can handle - very large number of request + """Send 500 requests at one to check if we can handle + very large number of request. """ def get_deferred(): @@ -451,7 +448,7 @@ class Https2ClientProtocolTestCase(TestCase): 200 ) - return self._check_repeat(get_deferred, 1000) + return self._check_repeat(get_deferred, 500) def test_inactive_stream(self): """Here we send 110 requests considering the MAX_CONCURRENT_STREAMS @@ -524,6 +521,10 @@ class Https2ClientProtocolTestCase(TestCase): def assert_metadata(response: Response): self.assertEqual(response.request, request) self.assertIsInstance(response.certificate, Certificate) + self.assertIsNotNone(response.certificate.original) + self.assertEqual(response.certificate.getIssuer(), self.client_certificate.getIssuer()) + self.assertTrue(response.certificate.getPublicKey().matches(self.client_certificate.getPublicKey())) + self.assertIsInstance(response.ip_address, IPv4Address) self.assertEqual(str(response.ip_address), '127.0.0.1') @@ -532,3 +533,40 @@ class Https2ClientProtocolTestCase(TestCase): d.addErrback(self.fail) return d + + def _check_invalid_netloc(self, url): + request = Request(url) + + def assert_invalid_hostname(failure: Failure): + self.assertIsNotNone(failure.check(InvalidHostname)) + error_msg = str(failure.value) + self.assertIn('localhost', error_msg) + self.assertIn('127.0.0.1', error_msg) + self.assertIn(str(request), error_msg) + + d = self.client.request(request) + d.addCallback(self.fail) + d.addErrback(assert_invalid_hostname) + return d + + def test_invalid_hostname(self): + return self._check_invalid_netloc('https://notlocalhost.notlocalhostdomain') + + def test_invalid_host_port(self): + port = self.port_number + 1 + return self._check_invalid_netloc(f'https://127.0.0.1:{port}') + + def test_connection_stays_with_invalid_requests(self): + d_list = [ + self.test_invalid_hostname(), + self.test_invalid_host_port(), + self._check_GET(Request(self.get_url('/get-data-html-small')), Data.HTML_SMALL, 200), + self._check_POST_json( + JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL), + Data.JSON_SMALL, + Data.EXTRA_SMALL, + 200 + ) + ] + + return DeferredList(d_list, fireOnOneErrback=True) From 4acdc2e5d623c6330235647aaad817e77eaad800 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 1 Jul 2020 20:15:33 +0530 Subject: [PATCH 023/114] refactor: use __qualname__, () for large strings --- scrapy/core/http2/protocol.py | 2 +- scrapy/core/http2/stream.py | 24 +++++++++++++++--------- tests/test_http2_client_protocol.py | 9 ++------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 5de516482..a3dfdb76e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -121,7 +121,7 @@ class H2ClientProtocol(Protocol): def request(self, request: Request): if not isinstance(request, Request): - raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__name__}') + raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}') stream = self._new_stream(request) d = stream.get_response() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index cc751b682..f45ddc04e 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -146,7 +146,7 @@ class Stream: self._deferred_response = Deferred(_cancel) def __str__(self): - return f'Stream(id={self.stream_id})' + return f'Stream(id={self.stream_id!r})' __repr__ = __str__ @@ -190,11 +190,11 @@ class Stream: def _get_request_headers(self): url = urlparse(self._request.url) - # Make sure pseudo-headers comes before all the other headers path = url.path if url.query: path += '?' + url.query + # Make sure pseudo-headers comes before all the other headers headers = [ (':method', self._request.method), (':authority', url.netloc), @@ -284,8 +284,10 @@ class Stream: if self._log_warnsize: self._reached_warnsize = True - warning_msg = f"Received more ({self._response['flow_controlled_size']}) bytes than download " \ - + f'warn size ({self._download_warnsize}) in request {self._request}' + warning_msg = ( + f'Received more ({self._response["flow_controlled_size"]}) bytes than download ' + f'warn size ({self._download_warnsize}) in request {self._request}' + ) logger.warning(warning_msg) # Acknowledge the data received @@ -306,8 +308,10 @@ class Stream: if self._log_warnsize: self._reached_warnsize = True - warning_msg = f'Expected response size ({expected_size}) larger than ' \ - + f'download warn size ({self._download_warnsize}) in request {self._request}' + warning_msg = ( + f'Expected response size ({expected_size}) larger than ' + f'download warn size ({self._download_warnsize}) in request {self._request}' + ) logger.warning(warning_msg) def reset_stream(self, reason=StreamCloseReason.RESET): @@ -340,7 +344,7 @@ class Stream: raise StreamClosedError(self.stream_id) if not isinstance(reason, StreamCloseReason): - raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__name__}') + raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}') self._cb_close(self.stream_id) self.stream_closed_server = True @@ -356,8 +360,10 @@ class Stream: # having Content-Length) if reason is StreamCloseReason.MAXSIZE_EXCEEDED: expected_size = int(self._response['headers'].get(b'Content-Length', -1)) - error_msg = f'Cancelling download of {self._request.url}: expected response ' \ - f'size ({expected_size}) larger than download max size ({self._download_maxsize}).' + error_msg = ( + f'Cancelling download of {self._request.url}: expected response ' + f'size ({expected_size}) larger than download max size ({self._download_maxsize}).' + ) logger.error(error_msg) self._deferred_response.errback(CancelledError(error_msg)) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 8f9fbe6fd..ca8a629b1 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -560,13 +560,8 @@ class Https2ClientProtocolTestCase(TestCase): d_list = [ self.test_invalid_hostname(), self.test_invalid_host_port(), - self._check_GET(Request(self.get_url('/get-data-html-small')), Data.HTML_SMALL, 200), - self._check_POST_json( - JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL), - Data.JSON_SMALL, - Data.EXTRA_SMALL, - 200 - ) + self.test_GET_small_body(), + self.test_POST_small_json() ] return DeferredList(d_list, fireOnOneErrback=True) From a94b30342a451b94e7f358f68ce1b1adc20723f9 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 6 Jul 2020 12:49:12 +0530 Subject: [PATCH 024/114] test: reduce test data size to 1MB --- scrapy/core/http2/stream.py | 2 -- tests/test_http2_client_protocol.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index f45ddc04e..1c856ff68 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -243,7 +243,6 @@ class Stream: bytes_to_send_size = min(window_size, self.remaining_content_length) # We now need to send a number of data frames. - data_frames_sent = 0 while bytes_to_send_size > 0: chunk_size = min(bytes_to_send_size, max_frame_size) @@ -252,7 +251,6 @@ class Stream: self._conn.send_data(self.stream_id, data_chunk, end_stream=False) - data_frames_sent += 1 bytes_to_send_size = bytes_to_send_size - chunk_size self.remaining_content_length = self.remaining_content_length - chunk_size diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index ca8a629b1..98dc98a0f 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -42,8 +42,8 @@ def make_html_body(val): class Data: - SMALL_SIZE = 1024 * 10 # 10 KB - LARGE_SIZE = (1024 ** 2) * 10 # 10 MB + SMALL_SIZE = 1024 # 1 KB + LARGE_SIZE = 1024 ** 2 # 1 MB STR_SMALL = generate_random_string(SMALL_SIZE) STR_LARGE = generate_random_string(LARGE_SIZE) From 7f5bb6b34c6a5137aa12cd4ad4f3845a8c67653f Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 6 Jul 2020 13:08:14 +0530 Subject: [PATCH 025/114] chore: add h2 to setup.py, tox.ini - Change log level for hpack to ERROR --- scrapy/utils/log.py | 3 +++ setup.py | 3 ++- tox.ini | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 51d276097..4e2671478 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -52,6 +52,9 @@ DEFAULT_LOGGING = { 'twisted': { 'level': 'ERROR', }, + 'hpack': { + 'level': 'ERROR', + }, } } diff --git a/setup.py b/setup.py index 47c5906e4..d872d6472 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,8 @@ setup( 'zope.interface>=4.1.3', 'protego>=0.1.15', 'itemadapter>=0.1.0', - 'typing_extensions>=3.7' + 'typing_extensions>=3.7', + 'h2>=3.2.0' ], extras_require=extras_require, ) \ No newline at end of file diff --git a/tox.ini b/tox.ini index ada211b3c..bc6314a2f 100644 --- a/tox.ini +++ b/tox.ini @@ -84,6 +84,7 @@ deps = # Extras botocore==1.3.23 Pillow==3.4.2 + h2==3.2.0 [testenv:extra-deps] deps = From 54e4228c3a22164b79db36a29a5e2d64391b592a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 6 Jul 2020 14:10:45 -0300 Subject: [PATCH 026/114] refactor: use protocol - H2ClientProtocol.close_stream - Fix and add missing type hints - More adjustments - Rename stream id generator - Simplify decrement --- scrapy/core/http2/protocol.py | 92 +++++++++++++++--------------- scrapy/core/http2/stream.py | 103 ++++++++++++++++------------------ 2 files changed, 93 insertions(+), 102 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index a3dfdb76e..2f177656d 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -11,32 +11,34 @@ from h2.events import ( StreamEnded, StreamReset, WindowUpdated ) from h2.exceptions import ProtocolError +from twisted.internet.defer import Deferred from twisted.internet.protocol import connectionDone, Protocol from twisted.internet.ssl import Certificate +from twisted.python.failure import Failure from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request + logger = logging.getLogger(__name__) class H2ClientProtocol(Protocol): - def __init__(self): + def __init__(self) -> None: config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) # ID of the next request stream - # Following the convention made by hyper-h2 each client ID - # will be odd. - self.stream_id_count = itertools.count(start=1, step=2) + # Following the convention made by hyper-h2 all IDs will be odd + self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs self.streams: Dict[int, Stream] = {} # If requests are received before connection is made we keep # all requests in a pool and send them as the connection is made - self._pending_request_stream_pool = deque() + self._pending_request_stream_pool: deque = deque() # Counter to keep track of opened stream. This counter # is used to make sure that not more than MAX_CONCURRENT_STREAMS @@ -48,15 +50,15 @@ class H2ClientProtocol(Protocol): # We pass this instance to the streams ResponseFailed() failure self._protocol_error: Optional[ProtocolError] = None - self._metadata: H2ConnectionMetadataDict = { + self.metadata: H2ConnectionMetadataDict = { 'certificate': None, 'ip_address': None, 'hostname': None, - 'port': None + 'port': None, } @property - def is_connected(self): + def is_connected(self) -> bool: """Boolean to keep track of the connection status. This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection @@ -75,7 +77,7 @@ class H2ClientProtocol(Protocol): self.conn.remote_settings.max_concurrent_streams ) - def _send_pending_requests(self): + def _send_pending_requests(self) -> None: """Initiate all pending requests from the deque following FIFO We make sure that at any time {allowed_max_concurrent_streams} streams are active. @@ -89,37 +91,33 @@ class H2ClientProtocol(Protocol): stream = self._pending_request_stream_pool.popleft() stream.initiate_request() - def _stream_close_cb(self, stream_id: int): - """Called when stream is closed completely + def pop_stream(self, stream_id: int) -> Stream: + """Perform cleanup when a stream is closed """ - self.streams.pop(stream_id) + stream = self.streams.pop(stream_id) self._active_streams -= 1 self._send_pending_requests() + return stream - def _new_stream(self, request: Request): + def _new_stream(self, request: Request) -> Stream: """Instantiates a new Stream object """ - stream_id = next(self.stream_id_count) - stream = Stream( - stream_id=stream_id, + stream_id=next(self._stream_id_generator), request=request, - connection=self.conn, - conn_metadata=self._metadata, - cb_close=self._stream_close_cb + protocol=self, ) - self.streams[stream.stream_id] = stream return stream - def _write_to_transport(self): + def _write_to_transport(self) -> None: """ Write data to the underlying transport connection from the HTTP2 connection instance if any """ data = self.conn.data_to_send() self.transport.write(data) - def request(self, request: Request): + def request(self, request: Request) -> Deferred: if not isinstance(request, Request): raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}') @@ -130,20 +128,20 @@ class H2ClientProtocol(Protocol): self._pending_request_stream_pool.append(stream) return d - def connectionMade(self): + def connectionMade(self) -> None: """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ destination = self.transport.getPeer() logger.debug('Connection made to {}'.format(destination)) - self._metadata['ip_address'] = ipaddress.ip_address(destination.host) - self._metadata['port'] = destination.port - self._metadata['hostname'] = self.transport.transport.addr[0] + self.metadata['ip_address'] = ipaddress.ip_address(destination.host) + self.metadata['port'] = destination.port + self.metadata['hostname'] = self.transport.transport.addr[0] self.conn.initiate_connection() self._write_to_transport() - def dataReceived(self, data): + def dataReceived(self, data: bytes) -> None: try: events = self.conn.receive_data(data) self._handle_events(events) @@ -158,32 +156,30 @@ class H2ClientProtocol(Protocol): finally: self._write_to_transport() - def connectionLost(self, reason=connectionDone): + def connectionLost(self, reason: Failure = connectionDone) -> None: """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ - # Pop all streams which were pending and were not yet started - # NOTE: Stream.close() pops the element from the streams dictionary - # which raises `RuntimeError: dictionary changed size during iteration` - # Hence, we copy the streams into a list. - for stream in list(self.streams.values()): + for stream in self.streams.values(): if stream.request_sent: - stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error) + stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error, from_protocol=True) else: - stream.close(StreamCloseReason.INACTIVE) + stream.close(StreamCloseReason.INACTIVE, from_protocol=True) + self._active_streams -= len(self.streams) + self.streams.clear() + self._send_pending_requests() self.conn.close_connection() if not reason.check(connectionDone): logger.warning("Connection lost with reason " + str(reason)) - def _handle_events(self, events): + def _handle_events(self, events: list) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler Arguments: - events {list} -- A list of events that the remote peer - triggered by sending data + events -- A list of events that the remote peer triggered by sending data """ for event in events: if isinstance(event, DataReceived): @@ -202,27 +198,29 @@ class H2ClientProtocol(Protocol): logger.debug('Received unhandled event {}'.format(event)) # Event handler functions starts here - def data_received(self, event: DataReceived): + def data_received(self, event: DataReceived) -> None: self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) - def response_received(self, event: ResponseReceived): + def response_received(self, event: ResponseReceived) -> None: self.streams[event.stream_id].receive_headers(event.headers) - def settings_acknowledged(self, event: SettingsAcknowledged): + def settings_acknowledged(self, event: SettingsAcknowledged) -> None: # Send off all the pending requests as now we have # established a proper HTTP/2 connection self._send_pending_requests() # Update certificate when our HTTP/2 connection is established - self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) + self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) - def stream_ended(self, event: StreamEnded): - self.streams[event.stream_id].close(StreamCloseReason.ENDED) + def stream_ended(self, event: StreamEnded) -> None: + stream = self.pop_stream(event.stream_id) + stream.close(StreamCloseReason.ENDED, from_protocol=True) - def stream_reset(self, event: StreamReset): - self.streams[event.stream_id].close(StreamCloseReason.RESET) + def stream_reset(self, event: StreamReset) -> None: + stream = self.pop_stream(event.stream_id) + stream.close(StreamCloseReason.RESET, from_protocol=True) - def window_updated(self, event: WindowUpdated): + def window_updated(self, event: WindowUpdated) -> None: if event.stream_id != 0: self.streams[event.stream_id].receive_window_update() else: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 1c856ff68..77cfbcfbf 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,22 +1,27 @@ import logging from enum import Enum from io import BytesIO -from typing import Callable, List +from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse -from h2.connection import H2Connection from h2.errors import ErrorCodes from h2.exceptions import StreamClosedError +from hpack import HeaderTuple from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from scrapy.core.http2.types import H2ConnectionMetadataDict, H2ResponseDict +from scrapy.core.http2.types import H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes + +if TYPE_CHECKING: + from scrapy.core.http2.protocol import H2ClientProtocol + + logger = logging.getLogger(__name__) @@ -31,12 +36,12 @@ class InactiveStreamClosed(ConnectionClosed): class InvalidHostname(Exception): - def __init__(self, request: Request, expected_hostname, expected_netloc): + def __init__(self, request: Request, expected_hostname: Optional[str], expected_netloc: Optional[str]) -> None: self.request = request self.expected_hostname = expected_hostname self.expected_netloc = expected_netloc - def __str__(self): + def __str__(self) -> str: return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}' @@ -80,28 +85,20 @@ class Stream: self, stream_id: int, request: Request, - connection: H2Connection, - conn_metadata: H2ConnectionMetadataDict, - cb_close: Callable[[int], None], + protocol: "H2ClientProtocol", download_maxsize: int = 0, download_warnsize: int = 0, fail_on_data_loss: bool = True - ): + ) -> None: """ Arguments: - stream_id -- For one HTTP/2 connection each stream is - uniquely identified by a single integer - request -- HTTP request - connection -- HTTP/2 connection this stream belongs to. - conn_metadata -- Reference to dictionary having metadata of HTTP/2 connection - cb_close -- Method called when this stream is closed - to notify the TCP connection instance. + stream_id -- Unique identifier for the stream within a single HTTP/2 connection + request -- The HTTP request associated to the stream + protocol -- Parent H2ClientProtocol instance """ - self.stream_id = stream_id - self._request = request - self._conn = connection - self._conn_metadata = conn_metadata - self._cb_close = cb_close + self.stream_id: int = stream_id + self._request: Request = request + self._protocol: "H2ClientProtocol" = protocol self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) @@ -132,7 +129,7 @@ class Stream: self._response: H2ResponseDict = { 'body': BytesIO(), 'flow_controlled_size': 0, - 'headers': Headers({}) + 'headers': Headers({}), } def _cancel(_): @@ -145,7 +142,7 @@ class Stream: self._deferred_response = Deferred(_cancel) - def __str__(self): + def __str__(self) -> str: return f'Stream(id={self.stream_id!r})' __repr__ = __str__ @@ -169,12 +166,9 @@ class Stream: and not self._reached_warnsize ) - def get_response(self): + def get_response(self) -> Deferred: """Simply return a Deferred which fires when response from the asynchronous request is available - - Returns: - Deferred -- Calls the callback passing the response """ return self._deferred_response @@ -182,12 +176,12 @@ class Stream: # Make sure that we are sending the request to the correct URL url = urlparse(self._request.url) return ( - url.netloc == self._conn_metadata['hostname'] - or url.netloc == f'{self._conn_metadata["hostname"]}:{self._conn_metadata["port"]}' - or url.netloc == f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}' + url.netloc == self._protocol.metadata['hostname'] + or url.netloc == f'{self._protocol.metadata["hostname"]}:{self._protocol.metadata["port"]}' + or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' ) - def _get_request_headers(self): + def _get_request_headers(self) -> List[Tuple[str, str]]: url = urlparse(self._request.url) path = url.path @@ -207,10 +201,10 @@ class Stream: return headers - def initiate_request(self): + def initiate_request(self) -> None: if self.check_request_url(): headers = self._get_request_headers() - self._conn.send_headers(self.stream_id, headers, end_stream=False) + self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False) self.request_sent = True self.send_data() else: @@ -218,7 +212,7 @@ class Stream: # Note that we have not sent any headers self.close(StreamCloseReason.INVALID_HOSTNAME) - def send_data(self): + def send_data(self) -> None: """Called immediately after the headers are sent. Here we send all the data as part of the request. @@ -233,10 +227,10 @@ class Stream: raise StreamClosedError(self.stream_id) # Firstly, check what the flow control window is for current stream. - window_size = self._conn.local_flow_control_window(stream_id=self.stream_id) + window_size = self._protocol.conn.local_flow_control_window(stream_id=self.stream_id) # Next, check what the maximum frame size is. - max_frame_size = self._conn.max_outbound_frame_size + max_frame_size = self._protocol.conn.max_outbound_frame_size # We will send no more than the window size or the remaining file size # of data in this call, whichever is smaller. @@ -249,7 +243,7 @@ class Stream: data_chunk_start_id = self.content_length - self.remaining_content_length data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size] - self._conn.send_data(self.stream_id, data_chunk, end_stream=False) + self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False) bytes_to_send_size = bytes_to_send_size - chunk_size self.remaining_content_length = self.remaining_content_length - chunk_size @@ -258,12 +252,12 @@ class Stream: # End the stream if no more data needs to be send if self.remaining_content_length == 0: - self._conn.end_stream(self.stream_id) + self._protocol.conn.end_stream(self.stream_id) # Q. What about the rest of the data? # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame - def receive_window_update(self): + def receive_window_update(self) -> None: """Flow control window size was changed. Send data that earlier could not be sent as we were blocked behind the flow control. @@ -271,7 +265,7 @@ class Stream: if self.remaining_content_length and not self.stream_closed_server and self.request_sent: self.send_data() - def receive_data(self, data: bytes, flow_controlled_length: int): + def receive_data(self, data: bytes, flow_controlled_length: int) -> None: self._response['body'].write(data) self._response['flow_controlled_size'] += flow_controlled_length @@ -289,12 +283,12 @@ class Stream: logger.warning(warning_msg) # Acknowledge the data received - self._conn.acknowledge_received_data( + self._protocol.conn.acknowledge_received_data( self._response['flow_controlled_size'], self.stream_id ) - def receive_headers(self, headers): + def receive_headers(self, headers: List[HeaderTuple]) -> None: for name, value in headers: self._response['headers'][name] = value @@ -312,7 +306,7 @@ class Stream: ) logger.warning(warning_msg) - def reset_stream(self, reason=StreamCloseReason.RESET): + def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None: """Close this stream by sending a RST_FRAME to the remote peer""" if self.stream_closed_local: raise StreamClosedError(self.stream_id) @@ -321,7 +315,7 @@ class Stream: self._response['body'].truncate(0) self.stream_closed_local = True - self._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) + self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) self.close(reason) def _is_data_lost(self) -> bool: @@ -332,11 +326,8 @@ class Stream: return expected_size != received_body_size - def close(self, reason: StreamCloseReason, error: Exception = None): + def close(self, reason: StreamCloseReason, error: Optional[Exception] = None, from_protocol: bool = False) -> None: """Based on the reason sent we will handle each case. - - Arguments: - reason -- One if StreamCloseReason """ if self.stream_closed_server: raise StreamClosedError(self.stream_id) @@ -344,7 +335,9 @@ class Stream: if not isinstance(reason, StreamCloseReason): raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}') - self._cb_close(self.stream_id) + if not from_protocol: + self._protocol.pop_stream(self.stream_id) + self.stream_closed_server = True flags = None @@ -392,11 +385,11 @@ class Stream: elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( self._request, - self._conn_metadata['hostname'], - f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}' + self._protocol.metadata['hostname'], + f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' )) - def _fire_response_deferred(self, flags: List[str] = None): + def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None: """Builds response from the self._response dict and fires the response deferred callback with the generated response instance""" @@ -405,7 +398,7 @@ class Stream: response_cls = responsetypes.from_args( headers=self._response['headers'], url=self._request.url, - body=body + body=body, ) response = response_cls( @@ -415,8 +408,8 @@ class Stream: body=body, request=self._request, flags=flags, - certificate=self._conn_metadata['certificate'], - ip_address=self._conn_metadata['ip_address'] + certificate=self._protocol.metadata['certificate'], + ip_address=self._protocol.metadata['ip_address'], ) self._deferred_response.callback(response) From 1c40dfa7408026bc9ae831000a8614e8f4a9dd0d Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 7 Jul 2020 00:44:09 +0530 Subject: [PATCH 027/114] fix: handle CONNECTION_LOST & RESET separately --- scrapy/core/http2/protocol.py | 19 ++++++++++++------- scrapy/core/http2/stream.py | 18 ++++++++++++------ scrapy/utils/log.py | 6 +++--- tests/test_http2_client_protocol.py | 2 +- tox.ini | 2 +- 5 files changed, 29 insertions(+), 18 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 2f177656d..55dbcabec 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -20,7 +20,6 @@ from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request - logger = logging.getLogger(__name__) @@ -30,7 +29,8 @@ class H2ClientProtocol(Protocol): self.conn = H2Connection(config=config) # ID of the next request stream - # Following the convention made by hyper-h2 all IDs will be odd + # Following the convention - 'Streams initiated by a client MUST + # use odd-numbered stream identifiers' (RFC 7540) self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs @@ -160,20 +160,25 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ + errors = [] + if not reason.check(connectionDone): + logger.warning("Connection lost with reason " + str(reason)) + errors.append(reason) + + if self._protocol_error: + errors.append(self._protocol_error) + for stream in self.streams.values(): if stream.request_sent: - stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error, from_protocol=True) + stream.close(StreamCloseReason.CONNECTION_LOST, errors, from_protocol=True) else: stream.close(StreamCloseReason.INACTIVE, from_protocol=True) self._active_streams -= len(self.streams) self.streams.clear() - self._send_pending_requests() + self._pending_request_stream_pool.clear() self.conn.close_connection() - if not reason.check(connectionDone): - logger.warning("Connection lost with reason " + str(reason)) - def _handle_events(self, events: list) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 77cfbcfbf..8b66d4b85 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -17,11 +17,9 @@ from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes - if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol - logger = logging.getLogger(__name__) @@ -326,7 +324,12 @@ class Stream: return expected_size != received_body_size - def close(self, reason: StreamCloseReason, error: Optional[Exception] = None, from_protocol: bool = False) -> None: + def close( + self, + reason: StreamCloseReason, + errors: Optional[List[Exception]] = None, + from_protocol: bool = False + ) -> None: """Based on the reason sent we will handle each case. """ if self.stream_closed_server: @@ -374,11 +377,14 @@ class Stream: self._response['headers'][':status'] = '499' self._fire_response_deferred() - elif reason in (StreamCloseReason.RESET, StreamCloseReason.CONNECTION_LOST): + elif reason is StreamCloseReason.RESET: self._deferred_response.errback(ResponseFailed([ - error if error else Failure() + Failure(f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM') ])) + elif reason is StreamCloseReason.CONNECTION_LOST: + self._deferred_response.errback(ResponseFailed(errors)) + elif reason is StreamCloseReason.INACTIVE: self._deferred_response.errback(InactiveStreamClosed(self._request)) @@ -403,7 +409,7 @@ class Stream: response = response_cls( url=self._request.url, - status=self._response['headers'][':status'], + status=int(self._response['headers'][':status']), headers=self._response['headers'], body=body, request=self._request, diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 4e2671478..9d59fdd68 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -46,15 +46,15 @@ DEFAULT_LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'loggers': { + 'hpack': { + 'level': 'ERROR', + }, 'scrapy': { 'level': 'DEBUG', }, 'twisted': { 'level': 'ERROR', }, - 'hpack': { - 'level': 'ERROR', - }, } } diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 98dc98a0f..9efca5267 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -474,7 +474,7 @@ class Https2ClientProtocolTestCase(TestCase): # Close the connection now to fire all the extra 10 requests errback # with InactiveStreamClosed - self.client.transport.abortConnection() + self.client.transport.loseConnection() return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) diff --git a/tox.ini b/tox.ini index bc6314a2f..be3b56e2d 100644 --- a/tox.ini +++ b/tox.ini @@ -83,8 +83,8 @@ deps = -rtests/requirements-py3.txt # Extras botocore==1.3.23 - Pillow==3.4.2 h2==3.2.0 + Pillow==3.4.2 [testenv:extra-deps] deps = From 2ea7d82534cafe5b25b28ef5a78e5b714767d27a Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 8 Jul 2020 18:57:13 +0530 Subject: [PATCH 028/114] feat: H2ClientFactory --- scrapy/core/http2/protocol.py | 35 +++++++++++++++++++++-------- scrapy/core/http2/stream.py | 29 +++++++++++++----------- scrapy/core/http2/types.py | 13 ++++++----- tests/test_http2_client_protocol.py | 8 ++++--- 4 files changed, 55 insertions(+), 30 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 55dbcabec..ee51300cc 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -2,29 +2,38 @@ import ipaddress import itertools import logging from collections import deque -from typing import Dict, Optional +from typing import Dict, List, Optional from h2.config import H2Configuration from h2.connection import H2Connection from h2.events import ( - DataReceived, ResponseReceived, SettingsAcknowledged, + Event, DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) from h2.exceptions import ProtocolError from twisted.internet.defer import Deferred -from twisted.internet.protocol import connectionDone, Protocol +from twisted.internet.protocol import connectionDone, Factory, Protocol from twisted.internet.ssl import Certificate from twisted.python.failure import Failure +from twisted.web.client import URI from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request +from scrapy.settings import Settings logger = logging.getLogger(__name__) class H2ClientProtocol(Protocol): - def __init__(self) -> None: + def __init__(self, uri: URI, settings: Settings) -> None: + """ + Arguments: + uri -- URI of the base url to which HTTP/2 Connection will be made. + uri is used to verify that incoming client requests have correct + base URL. + settings -- Scrapy project settings + """ config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) @@ -53,8 +62,9 @@ class H2ClientProtocol(Protocol): self.metadata: H2ConnectionMetadataDict = { 'certificate': None, 'ip_address': None, - 'hostname': None, - 'port': None, + 'uri': uri, + 'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'), + 'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'), } @property @@ -135,8 +145,6 @@ class H2ClientProtocol(Protocol): destination = self.transport.getPeer() logger.debug('Connection made to {}'.format(destination)) self.metadata['ip_address'] = ipaddress.ip_address(destination.host) - self.metadata['port'] = destination.port - self.metadata['hostname'] = self.transport.transport.addr[0] self.conn.initiate_connection() self._write_to_transport() @@ -179,7 +187,7 @@ class H2ClientProtocol(Protocol): self._pending_request_stream_pool.clear() self.conn.close_connection() - def _handle_events(self, events: list) -> None: + def _handle_events(self, events: List[Event]) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler @@ -232,3 +240,12 @@ class H2ClientProtocol(Protocol): # Send leftover data for all the streams for stream in self.streams.values(): stream.receive_window_update() + + +class H2ClientFactory(Factory): + def __init__(self, uri: URI, settings: Settings) -> None: + self.uri = uri + self.settings = settings + + def buildProtocol(self, addr) -> H2ClientProtocol: + return H2ClientProtocol(self.uri, self.settings) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 8b66d4b85..5017f9cd4 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -16,6 +16,7 @@ from scrapy.core.http2.types import H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes +from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -34,7 +35,7 @@ class InactiveStreamClosed(ConnectionClosed): class InvalidHostname(Exception): - def __init__(self, request: Request, expected_hostname: Optional[str], expected_netloc: Optional[str]) -> None: + def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None: self.request = request self.expected_hostname = expected_hostname self.expected_netloc = expected_netloc @@ -83,10 +84,7 @@ class Stream: self, stream_id: int, request: Request, - protocol: "H2ClientProtocol", - download_maxsize: int = 0, - download_warnsize: int = 0, - fail_on_data_loss: bool = True + protocol: "H2ClientProtocol" ) -> None: """ Arguments: @@ -98,9 +96,14 @@ class Stream: self._request: Request = request self._protocol: "H2ClientProtocol" = protocol - self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) - self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) - self._fail_on_dataloss = self._request.meta.get('download_fail_on_dataloss', fail_on_data_loss) + self._download_maxsize = self._request.meta.get( + 'download_maxsize', + self._protocol.metadata['default_download_maxsize'] + ) + self._download_warnsize = self._request.meta.get( + 'download_warnsize', + self._protocol.metadata['default_download_warnsize'] + ) self.request_start_time = None @@ -174,9 +177,9 @@ class Stream: # Make sure that we are sending the request to the correct URL url = urlparse(self._request.url) return ( - url.netloc == self._protocol.metadata['hostname'] - or url.netloc == f'{self._protocol.metadata["hostname"]}:{self._protocol.metadata["port"]}' - or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' + url.netloc == to_unicode(self._protocol.metadata['uri'].host) + or url.netloc == to_unicode(self._protocol.metadata['uri'].netloc) + or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' ) def _get_request_headers(self) -> List[Tuple[str, str]]: @@ -391,8 +394,8 @@ class Stream: elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( self._request, - self._protocol.metadata['hostname'], - f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' + to_unicode(self._protocol.metadata['uri'].host), + f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' )) def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None: diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py index dd7b1187b..d2aa1a9d8 100644 --- a/scrapy/core/http2/types.py +++ b/scrapy/core/http2/types.py @@ -3,6 +3,7 @@ from ipaddress import IPv4Address, IPv6Address from typing import Union, Optional from twisted.internet.ssl import Certificate +from twisted.web.client import URI # for python < 3.8 -- typing.TypedDict is undefined from typing_extensions import TypedDict @@ -19,14 +20,16 @@ class H2ConnectionMetadataDict(TypedDict): # is updated when HTTP/2 connection is made successfully ip_address: Optional[Union[IPv4Address, IPv6Address]] - # Name of the peer HTTP/2 connection is established - hostname: Optional[str] + # URI of the peer HTTP/2 connection is made + uri: URI - port: Optional[int] - - # Both ip_address and hostname are used by the Stream before + # Both ip_address and uri are used by the Stream before # initiating the request to verify that the base address + # Variables taken from Project Settings + default_download_maxsize: int + default_download_warnsize: int + class H2ResponseDict(TypedDict): # Data received frame by frame from the server is appended diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 9efca5267..05e5f5047 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -11,17 +11,18 @@ from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint -from twisted.internet.protocol import Factory from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase +from twisted.web.client import URI from twisted.web.http import Request as TxRequest from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File -from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.core.http2.protocol import H2ClientFactory from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest +from scrapy.settings import Settings from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -183,7 +184,8 @@ class Https2ClientProtocolTestCase(TestCase): trustRoot=self.client_certificate, acceptableProtocols=[b'h2'] ) - h2_client_factory = Factory.forProtocol(H2ClientProtocol) + uri = URI.fromBytes(to_bytes(self.get_url('/'))) + h2_client_factory = H2ClientFactory(uri, Settings()) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) From 64c6af10e1b276db68ea722845621912d2023a75 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 13 Jul 2020 00:57:49 +0530 Subject: [PATCH 029/114] refactor: use str instead of to_unicode --- scrapy/core/http2/stream.py | 7 +++---- tests/test_http2_client_protocol.py | 29 ++++++++++++++++------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 5017f9cd4..a2f0e2aa4 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -16,7 +16,6 @@ from scrapy.core.http2.types import H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes -from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -177,8 +176,8 @@ class Stream: # Make sure that we are sending the request to the correct URL url = urlparse(self._request.url) return ( - url.netloc == to_unicode(self._protocol.metadata['uri'].host) - or url.netloc == to_unicode(self._protocol.metadata['uri'].netloc) + url.netloc == str(self._protocol.metadata['uri'].host, 'utf-8') + or url.netloc == str(self._protocol.metadata['uri'].netloc, 'utf-8') or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' ) @@ -394,7 +393,7 @@ class Stream: elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( self._request, - to_unicode(self._protocol.metadata['uri'].host), + str(self._protocol.metadata['uri'].host, 'utf-8'), f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' )) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 05e5f5047..05f4889ef 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -23,7 +23,6 @@ from scrapy.core.http2.protocol import H2ClientFactory from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.settings import Settings -from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -39,7 +38,7 @@ def make_html_body(val):

Hello from HTTP2

{val}

''' - return to_bytes(response) + return bytes(response, 'utf-8') class Data: @@ -83,10 +82,11 @@ class PostDataJsonMixin: 'extra-data': extra_data } for k, v in request.requestHeaders.getAllRawHeaders(): - response['request-headers'][to_unicode(k)] = to_unicode(v[0]) + response['request-headers'][str(k, 'utf-8')] = str(v[0], 'utf-8') - response_bytes = to_bytes(json.dumps(response)) - request.setHeader('Content-Type', 'application/json') + response_bytes = bytes(json.dumps(response), 'utf-8') + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') return response_bytes @@ -127,13 +127,14 @@ class NoContentLengthHeader(LeafResource): class QueryParams(LeafResource): def render_GET(self, request: TxRequest): - request.setHeader('Content-Type', 'application/json') + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') query_params = {} for k, v in request.args.items(): - query_params[to_unicode(k)] = to_unicode(v[0]) + query_params[str(k, 'utf-8')] = str(v[0], 'utf-8') - return to_bytes(json.dumps(query_params)) + return bytes(json.dumps(query_params), 'utf-8') def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: @@ -184,7 +185,7 @@ class Https2ClientProtocolTestCase(TestCase): trustRoot=self.client_certificate, acceptableProtocols=[b'h2'] ) - uri = URI.fromBytes(to_bytes(self.get_url('/'))) + uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) h2_client_factory = H2ClientFactory(uri, Settings()) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) @@ -278,7 +279,8 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(len(response.body), content_length) # Parse the body - body = json.loads(to_unicode(response.body)) + content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + body = json.loads(str(response.body, content_encoding)) self.assertIn('request-body', body) self.assertIn('extra-data', body) self.assertIn('request-headers', body) @@ -292,9 +294,9 @@ class Https2ClientProtocolTestCase(TestCase): # Check if headers were sent successfully request_headers = body['request-headers'] for k, v in request.headers.items(): - k_str = to_unicode(k) + k_str = str(k, 'utf-8') self.assertIn(k_str, request_headers) - self.assertEqual(request_headers[k_str], to_unicode(v[0])) + self.assertEqual(request_headers[k_str], str(v[0], 'utf-8')) d.addCallback(assert_response) d.addErrback(self.fail) @@ -494,7 +496,8 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(self.get_url(f'/query-params?{urlencode(params)}')) def assert_query_params(response: Response): - data = json.loads(to_unicode(response.body)) + content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + data = json.loads(str(response.body, content_encoding)) self.assertEqual(data, params) d = self.client.request(request) From aeaeb7385b7d1e6570bad38da4db492de8fb4206 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 14 Jul 2020 03:55:14 +0530 Subject: [PATCH 030/114] feat: assert negotiated protocol as h2 - implement IHandshakeListener in H2ClientProtocol to know when handshake is completed - implement IProtocolNegotiationFactory in H2ClientFactory to provide information about the acceptableProtols (h2) during NPN or ALPN protocol --- scrapy/core/http2/protocol.py | 69 ++++++++++++++++++++++------- scrapy/core/http2/stream.py | 4 +- tests/test_http2_client_protocol.py | 2 +- 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index ee51300cc..5d859d475 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -12,10 +12,12 @@ from h2.events import ( ) from h2.exceptions import ProtocolError from twisted.internet.defer import Deferred +from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory from twisted.internet.protocol import connectionDone, Factory, Protocol from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from twisted.web.client import URI +from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict @@ -25,15 +27,29 @@ from scrapy.settings import Settings logger = logging.getLogger(__name__) +class InvalidNegotiatedProtocol(ProtocolError): + + def __init__(self, negotiated_protocol: str) -> None: + self.negotiated_protocol = negotiated_protocol + + def __str__(self) -> str: + return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol}' + + +@implementer(IHandshakeListener) class H2ClientProtocol(Protocol): - def __init__(self, uri: URI, settings: Settings) -> None: + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: """ Arguments: uri -- URI of the base url to which HTTP/2 Connection will be made. uri is used to verify that incoming client requests have correct base URL. settings -- Scrapy project settings + conn_lost_deferred -- Deferred fires with the reason: Failure to notify + that connection was lost """ + self._conn_lost_deferred = conn_lost_deferred + config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) @@ -55,9 +71,9 @@ class H2ClientProtocol(Protocol): # We use simple FIFO policy to handle pending requests self._active_streams = 0 - # Save an instance of ProtocolError raised by hyper-h2 - # We pass this instance to the streams ResponseFailed() failure - self._protocol_error: Optional[ProtocolError] = None + # Save an instance of errors raised which lead to losing the connection + # We pass these instances to the streams ResponseFailed() failure + self._conn_lost_errors: List[BaseException] = [] self.metadata: H2ConnectionMetadataDict = { 'certificate': None, @@ -136,6 +152,12 @@ class H2ClientProtocol(Protocol): # Add the stream to the request pool self._pending_request_stream_pool.append(stream) + + # If we are connection and receive a request + # There is a good chance that the connection was IDLE + # Hence, we need to initiate pending requests + if self.is_connected: + self._send_pending_requests() return d def connectionMade(self) -> None: @@ -149,6 +171,19 @@ class H2ClientProtocol(Protocol): self.conn.initiate_connection() self._write_to_transport() + def _lose_connection_with_error(self, errors: List[BaseException]): + """Helper function to lose the connection with the error sent as a + reason""" + self._conn_lost_errors += errors + self.transport.loseConnection() + + def handshakeCompleted(self): + """We close the connection with InvalidNegotiatedProtocol exception + when the connection was not made via h2 protocol""" + negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') + if negotiated_protocol != 'h2': + self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) + def dataReceived(self, data: bytes) -> None: try: events = self.conn.receive_data(data) @@ -157,10 +192,7 @@ class H2ClientProtocol(Protocol): # Save this error as ultimately the connection will be dropped # internally by hyper-h2. Saved error will be passed to all the streams # closed with the connection. - self._protocol_error = e - - # We lose the transport connection here - self.transport.loseConnection() + self._lose_connection_with_error([e]) finally: self._write_to_transport() @@ -168,17 +200,17 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ - errors = [] + # Notify the connection pool instance such that no new requests are + # sent over current connection if not reason.check(connectionDone): - logger.warning("Connection lost with reason " + str(reason)) - errors.append(reason) + self._conn_lost_errors.append(reason) - if self._protocol_error: - errors.append(self._protocol_error) + if self._conn_lost_deferred: + self._conn_lost_deferred.callback(self._conn_lost_errors) for stream in self.streams.values(): if stream.request_sent: - stream.close(StreamCloseReason.CONNECTION_LOST, errors, from_protocol=True) + stream.close(StreamCloseReason.CONNECTION_LOST, self._conn_lost_errors, from_protocol=True) else: stream.close(StreamCloseReason.INACTIVE, from_protocol=True) @@ -242,10 +274,15 @@ class H2ClientProtocol(Protocol): stream.receive_window_update() +@implementer(IProtocolNegotiationFactory) class H2ClientFactory(Factory): - def __init__(self, uri: URI, settings: Settings) -> None: + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: self.uri = uri self.settings = settings + self.conn_lost_deferred = conn_lost_deferred def buildProtocol(self, addr) -> H2ClientProtocol: - return H2ClientProtocol(self.uri, self.settings) + return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) + + def acceptableProtocols(self) -> List[bytes]: + return [b'h2'] diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a2f0e2aa4..16d54b2fc 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -44,7 +44,7 @@ class InvalidHostname(Exception): class StreamCloseReason(Enum): - # Received a StreamEnded event + # Received a StreamEnded event from the remote ENDED = 1 # Received a StreamReset event -- ended abruptly @@ -329,7 +329,7 @@ class Stream: def close( self, reason: StreamCloseReason, - errors: Optional[List[Exception]] = None, + errors: Optional[List[BaseException]] = None, from_protocol: bool = False ) -> None: """Based on the reason sent we will handle each case. diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 05f4889ef..7fcea58c5 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -9,7 +9,7 @@ from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor -from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError +from twisted.internet.defer import CancelledError, DeferredList, inlineCallbacks from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure From 1dd27a92fa53be87c71df43bd6f3043a225a2c10 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 14 Jul 2020 17:58:22 +0530 Subject: [PATCH 031/114] feat: Idle Timeout for H2Connection (240s) --- scrapy/core/http2/protocol.py | 60 ++++++++++++++++++++++++----- scrapy/core/http2/stream.py | 3 +- tests/test_http2_client_protocol.py | 10 +++-- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 5d859d475..bf41a2805 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -6,15 +6,18 @@ from typing import Dict, List, Optional from h2.config import H2Configuration from h2.connection import H2Connection +from h2.errors import ErrorCodes from h2.events import ( Event, DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) from h2.exceptions import ProtocolError from twisted.internet.defer import Deferred +from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory from twisted.internet.protocol import connectionDone, Factory, Protocol from twisted.internet.ssl import Certificate +from twisted.protocols.policies import TimeoutMixin from twisted.python.failure import Failure from twisted.web.client import URI from zope.interface import implementer @@ -37,7 +40,9 @@ class InvalidNegotiatedProtocol(ProtocolError): @implementer(IHandshakeListener) -class H2ClientProtocol(Protocol): +class H2ClientProtocol(Protocol, TimeoutMixin): + IDLE_TIMEOUT = 240 + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: """ Arguments: @@ -71,6 +76,10 @@ class H2ClientProtocol(Protocol): # We use simple FIFO policy to handle pending requests self._active_streams = 0 + # Flag to keep track if settings were acknowledged by the remote + # This ensures that we have established a HTTP/2 connection + self._settings_acknowledged = False + # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure self._conn_lost_errors: List[BaseException] = [] @@ -84,12 +93,12 @@ class H2ClientProtocol(Protocol): } @property - def is_connected(self) -> bool: + def h2_connected(self) -> bool: """Boolean to keep track of the connection status. This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection """ - return bool(self.transport.connected) + return bool(self.transport.connected) and self._settings_acknowledged @property def allowed_max_concurrent_streams(self) -> int: @@ -111,7 +120,7 @@ class H2ClientProtocol(Protocol): while ( self._pending_request_stream_pool and self._active_streams < self.allowed_max_concurrent_streams - and self.is_connected + and self.h2_connected ): self._active_streams += 1 stream = self._pending_request_stream_pool.popleft() @@ -140,6 +149,9 @@ class H2ClientProtocol(Protocol): """ Write data to the underlying transport connection from the HTTP2 connection instance if any """ + # Reset the idle timeout as connection is still actively sending data + self.resetTimeout() + data = self.conn.data_to_send() self.transport.write(data) @@ -153,21 +165,23 @@ class H2ClientProtocol(Protocol): # Add the stream to the request pool self._pending_request_stream_pool.append(stream) - # If we are connection and receive a request - # There is a good chance that the connection was IDLE - # Hence, we need to initiate pending requests - if self.is_connected: - self._send_pending_requests() + # If we receive a request when connection is idle + # We need to initiate pending requests + self._send_pending_requests() return d def connectionMade(self) -> None: """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ + # Initialize the timeout + self.setTimeout(self.IDLE_TIMEOUT) + destination = self.transport.getPeer() logger.debug('Connection made to {}'.format(destination)) self.metadata['ip_address'] = ipaddress.ip_address(destination.host) + # Initiate H2 Connection self.conn.initiate_connection() self._write_to_transport() @@ -185,6 +199,9 @@ class H2ClientProtocol(Protocol): self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) def dataReceived(self, data: bytes) -> None: + # Reset the idle timeout as connection is still actively receiving data + self.resetTimeout() + try: events = self.conn.receive_data(data) self._handle_events(events) @@ -196,10 +213,33 @@ class H2ClientProtocol(Protocol): finally: self._write_to_transport() + def timeoutConnection(self): + """Called when the connection times out. + We lose the connection with TimeoutError""" + + # Check whether there are open streams. If there are, we're going to + # want to use the error code PROTOCOL_ERROR. If there aren't, use + # NO_ERROR. + if ( + self.conn.open_outbound_streams > 0 + or self.conn.open_inbound_streams > 0 + or self._active_streams > 0 + ): + error_code = ErrorCodes.PROTOCOL_ERROR + else: + error_code = ErrorCodes.NO_ERROR + self.conn.close_connection(error_code=error_code) + self._write_to_transport() + + self._lose_connection_with_error([TimeoutError("Hello")]) + def connectionLost(self, reason: Failure = connectionDone) -> None: """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ + # Cancel the timeout if not done yet + self.setTimeout(None) + # Notify the connection pool instance such that no new requests are # sent over current connection if not reason.check(connectionDone): @@ -250,6 +290,8 @@ class H2ClientProtocol(Protocol): self.streams[event.stream_id].receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: + self._settings_acknowledged = True + # Send off all the pending requests as now we have # established a proper HTTP/2 connection self._send_pending_requests() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 16d54b2fc..f60691945 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -134,7 +134,8 @@ class Stream: def _cancel(_): # Close this stream as gracefully as possible - # Check if the stream has started + # If the associated request is initiated we reset this stream + # else we directly call close() method if self.request_sent: self.reset_stream(StreamCloseReason.CANCELLED) else: diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 7fcea58c5..2833801e7 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -9,7 +9,7 @@ from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor -from twisted.internet.defer import CancelledError, DeferredList, inlineCallbacks +from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure @@ -174,6 +174,7 @@ class Https2ClientProtocolTestCase(TestCase): # Start server for testing self.hostname = u'localhost' context_factory = ssl_context_factory(self.key_file, self.certificate_file) + server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) self.server = yield server_endpoint.listen(self.site) self.port_number = self.server.getHost().port @@ -186,17 +187,20 @@ class Https2ClientProtocolTestCase(TestCase): acceptableProtocols=[b'h2'] ) uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) - h2_client_factory = H2ClientFactory(uri, Settings()) + + self.conn_closed_deferred = Deferred() + h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) @inlineCallbacks def tearDown(self): - if self.client.is_connected: + if self.client.connected: yield self.client.transport.loseConnection() yield self.client.transport.abortConnection() yield self.server.stopListening() shutil.rmtree(self.temp_directory) + self.conn_closed_deferred = None def get_url(self, path): """ From e662762e6ab2f53ff2e31e21f34c078590f65aa9 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 15 Jul 2020 03:45:32 +0530 Subject: [PATCH 032/114] chore: Handle ConnectionTerminated event --- scrapy/core/http2/protocol.py | 44 +++++++++++++++++++++++++++-------- scrapy/core/http2/stream.py | 4 ++-- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index bf41a2805..041908116 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -2,16 +2,18 @@ import ipaddress import itertools import logging from collections import deque -from typing import Dict, List, Optional +from ipaddress import IPv4Address, IPv6Address +from typing import Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection from h2.errors import ErrorCodes from h2.events import ( - Event, DataReceived, ResponseReceived, SettingsAcknowledged, - StreamEnded, StreamReset, WindowUpdated + Event, ConnectionTerminated, DataReceived, ResponseReceived, + SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived, + WindowUpdated ) -from h2.exceptions import ProtocolError +from h2.exceptions import H2Error, ProtocolError from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory @@ -30,7 +32,7 @@ from scrapy.settings import Settings logger = logging.getLogger(__name__) -class InvalidNegotiatedProtocol(ProtocolError): +class InvalidNegotiatedProtocol(H2Error): def __init__(self, negotiated_protocol: str) -> None: self.negotiated_protocol = negotiated_protocol @@ -39,6 +41,15 @@ class InvalidNegotiatedProtocol(ProtocolError): return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol}' +class RemoteTerminatedConnection(H2Error): + def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], event: ConnectionTerminated): + self.remote_ip_address = remote_ip_address + self.terminate_event = event + + def __str__(self) -> str: + return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address}' + + @implementer(IHandshakeListener) class H2ClientProtocol(Protocol, TimeoutMixin): IDLE_TIMEOUT = 240 @@ -194,8 +205,12 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def handshakeCompleted(self): """We close the connection with InvalidNegotiatedProtocol exception when the connection was not made via h2 protocol""" - negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') + negotiated_protocol = self.transport.negotiatedProtocol + if type(negotiated_protocol) is bytes: + negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') if negotiated_protocol != 'h2': + # Here we have not initiated the connection yet + # So, no need to send a GOAWAY frame to the remote self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) def dataReceived(self, data: bytes) -> None: @@ -231,7 +246,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.conn.close_connection(error_code=error_code) self._write_to_transport() - self._lose_connection_with_error([TimeoutError("Hello")]) + self._lose_connection_with_error([ + TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s") + ]) def connectionLost(self, reason: Failure = connectionDone) -> None: """Called by Twisted when the transport connection is lost. @@ -267,7 +284,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin): events -- A list of events that the remote peer triggered by sending data """ for event in events: - if isinstance(event, DataReceived): + if isinstance(event, ConnectionTerminated): + self.connection_terminated(event) + elif isinstance(event, DataReceived): self.data_received(event) elif isinstance(event, ResponseReceived): self.response_received(event) @@ -279,10 +298,15 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.window_updated(event) elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) - else: - logger.debug('Received unhandled event {}'.format(event)) + elif isinstance(event, UnknownFrameReceived): + logger.debug(f'UnknownFrameReceived: frame={event.frame}') # Event handler functions starts here + def connection_terminated(self, event: ConnectionTerminated) -> None: + self._lose_connection_with_error([ + RemoteTerminatedConnection(self.metadata['ip_address'], event) + ]) + def data_received(self, event: DataReceived) -> None: self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index f60691945..15f081cab 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -5,7 +5,7 @@ from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse from h2.errors import ErrorCodes -from h2.exceptions import StreamClosedError +from h2.exceptions import H2Error, StreamClosedError from hpack import HeaderTuple from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed @@ -32,7 +32,7 @@ class InactiveStreamClosed(ConnectionClosed): self.request = request -class InvalidHostname(Exception): +class InvalidHostname(H2Error): def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None: self.request = request From 316620b517207b1082dd9c5b4ebfc7fbe745e3bb Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 22 Jul 2020 13:53:46 +0530 Subject: [PATCH 033/114] chore: pass spider as argument for request method - download_maxsize and download_warnsize can now be extracted from the spider directly and passed to the stream - remove `partial` flag from the response as per RFC 7540 - Section 8.1.2.6 --- scrapy/core/http2/protocol.py | 12 +++++--- scrapy/core/http2/stream.py | 35 ++++++++++------------ tests/test_http2_client_protocol.py | 46 +++++++++++++++++++---------- 3 files changed, 55 insertions(+), 38 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 041908116..feb034a0c 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -28,6 +28,7 @@ from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request from scrapy.settings import Settings +from scrapy.spiders import Spider logger = logging.getLogger(__name__) @@ -71,7 +72,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # ID of the next request stream # Following the convention - 'Streams initiated by a client MUST - # use odd-numbered stream identifiers' (RFC 7540) + # use odd-numbered stream identifiers' (RFC 7540 - Section 5.1.1) self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs @@ -136,6 +137,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._active_streams += 1 stream = self._pending_request_stream_pool.popleft() stream.initiate_request() + self._write_to_transport() def pop_stream(self, stream_id: int) -> Stream: """Perform cleanup when a stream is closed @@ -145,13 +147,15 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._send_pending_requests() return stream - def _new_stream(self, request: Request) -> Stream: + def _new_stream(self, request: Request, spider: Spider) -> Stream: """Instantiates a new Stream object """ stream = Stream( stream_id=next(self._stream_id_generator), request=request, protocol=self, + download_maxsize=getattr(spider, 'download_maxsize', self.metadata['default_download_maxsize']), + download_warnsize=getattr(spider, 'download_warnsize', self.metadata['default_download_warnsize']), ) self.streams[stream.stream_id] = stream return stream @@ -166,11 +170,11 @@ class H2ClientProtocol(Protocol, TimeoutMixin): data = self.conn.data_to_send() self.transport.write(data) - def request(self, request: Request) -> Deferred: + def request(self, request: Request, spider: Spider) -> Deferred: if not isinstance(request, Request): raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}') - stream = self._new_stream(request) + stream = self._new_stream(request, spider) d = stream.get_response() # Add the stream to the request pool diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 15f081cab..133196797 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -83,7 +83,9 @@ class Stream: self, stream_id: int, request: Request, - protocol: "H2ClientProtocol" + protocol: "H2ClientProtocol", + download_maxsize: int = 0, + download_warnsize: int = 0 ) -> None: """ Arguments: @@ -95,14 +97,8 @@ class Stream: self._request: Request = request self._protocol: "H2ClientProtocol" = protocol - self._download_maxsize = self._request.meta.get( - 'download_maxsize', - self._protocol.metadata['default_download_maxsize'] - ) - self._download_warnsize = self._request.meta.get( - 'download_warnsize', - self._protocol.metadata['default_download_warnsize'] - ) + self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) + self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) self.request_start_time = None @@ -346,26 +342,28 @@ class Stream: self.stream_closed_server = True - flags = None - if b'Content-Length' not in self._response['headers']: - # Missing Content-Length - {twisted.web.http.PotentialDataLoss} - flags = ['partial'] + # We do not check for Content-Length or Transfer-Encoding in response headers + # and add `partial` flag as in HTTP/1.1 as 'A request or response that includes + # a payload body can include a content-length header field' (RFC 7540 - Section 8.1.2.6) # NOTE: Order of handling the events is important here # As we immediately cancel the request when maxsize is exceeded while # receiving DATA_FRAME's when we have received the headers (not # having Content-Length) if reason is StreamCloseReason.MAXSIZE_EXCEEDED: - expected_size = int(self._response['headers'].get(b'Content-Length', -1)) + expected_size = int(self._response['headers'].get( + b'Content-Length', + self._response['flow_controlled_size']) + ) error_msg = ( - f'Cancelling download of {self._request.url}: expected response ' - f'size ({expected_size}) larger than download max size ({self._download_maxsize}).' + f'Cancelling download of {self._request.url}: received response ' + f'size ({expected_size}) larger than download max size ({self._download_maxsize})' ) logger.error(error_msg) self._deferred_response.errback(CancelledError(error_msg)) elif reason is StreamCloseReason.ENDED: - self._fire_response_deferred(flags) + self._fire_response_deferred() # Stream was abruptly ended here elif reason is StreamCloseReason.CANCELLED: @@ -398,7 +396,7 @@ class Stream: f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' )) - def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None: + def _fire_response_deferred(self) -> None: """Builds response from the self._response dict and fires the response deferred callback with the generated response instance""" @@ -416,7 +414,6 @@ class Stream: headers=self._response['headers'], body=body, request=self._request, - flags=flags, certificate=self._protocol.metadata['certificate'], ip_address=self._protocol.metadata['ip_address'], ) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 2833801e7..d0386f7f8 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -23,6 +23,7 @@ from scrapy.core.http2.protocol import H2ClientFactory from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.settings import Settings +from scrapy.spiders import Spider from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -41,6 +42,14 @@ def make_html_body(val): return bytes(response, 'utf-8') +class DummySpider(Spider): + name = 'dummy' + start_urls = [] + + def parse(self, response): + print(response) + + class Data: SMALL_SIZE = 1024 # 1 KB LARGE_SIZE = 1024 ** 2 # 1 MB @@ -210,6 +219,9 @@ class Https2ClientProtocolTestCase(TestCase): assert len(path) > 0 and (path[0] == '/' or path[0] == '&') return f'{self.scheme}://{self.hostname}:{self.port_number}{path}' + def make_request(self, request: Request) -> Deferred: + return self.client.request(request, DummySpider()) + @staticmethod def _check_repeat(get_deferred, count): d_list = [] @@ -233,7 +245,7 @@ class Https2ClientProtocolTestCase(TestCase): content_length = int(response.headers.get('Content-Length')) self.assertEqual(len(response.body), content_length) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(check_response) d.addErrback(self.fail) return d @@ -273,7 +285,7 @@ class Https2ClientProtocolTestCase(TestCase): expected_extra_data, expected_status: int ): - d = self.client.request(request) + d = self.make_request(request) def assert_response(response: Response): self.assertEqual(response.status, expected_status) @@ -355,7 +367,7 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, 499) self.assertEqual(response.request, request) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_response) d.addErrback(self.fail) d.cancel() @@ -367,8 +379,13 @@ class Https2ClientProtocolTestCase(TestCase): def assert_cancelled_error(failure): self.assertIsInstance(failure.value, CancelledError) + error_pattern = re.compile( + rf'Cancelling download of {request.url}: received response ' + rf'size \(\d*\) larger than download max size \(1000\)' + ) + self.assertEqual(len(re.findall(error_pattern, str(failure.value))), 1) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(self.fail) d.addErrback(assert_cancelled_error) return d @@ -385,7 +402,7 @@ class Https2ClientProtocolTestCase(TestCase): for error in failure.value.reasons )) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(self.fail) d.addErrback(assert_failure) return d @@ -397,10 +414,9 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, 200) self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) self.assertEqual(response.request, request) - self.assertIn('partial', response.flags) self.assertNotIn('Content-Length', response.headers) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_content_length) d.addErrback(self.fail) return d @@ -413,7 +429,7 @@ class Https2ClientProtocolTestCase(TestCase): expected_body ): with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm: - response = yield self.client.request(request) + response = yield self.make_request(request) self.assertEqual(response.status, 200) self.assertEqual(response.request, request) self.assertEqual(response.body, expected_body) @@ -469,13 +485,13 @@ class Https2ClientProtocolTestCase(TestCase): # Send 100 request (we do not check the result) for _ in range(100): - d = self.client.request(Request(self.get_url('/get-data-html-small'))) + d = self.make_request(Request(self.get_url('/get-data-html-small'))) d.addBoth(lambda _: None) d_list.append(d) # Now send 10 extra request and save the response deferred in a list for _ in range(10): - d = self.client.request(Request(self.get_url('/get-data-html-small'))) + d = self.make_request(Request(self.get_url('/get-data-html-small'))) d.addCallback(self.fail) d.addErrback(assert_inactive_stream) d_list.append(d) @@ -488,7 +504,7 @@ class Https2ClientProtocolTestCase(TestCase): def test_invalid_request_type(self): with self.assertRaises(TypeError): - self.client.request('https://InvalidDataTypePassed.com') + self.make_request('https://InvalidDataTypePassed.com') def test_query_parameters(self): params = { @@ -504,7 +520,7 @@ class Https2ClientProtocolTestCase(TestCase): data = json.loads(str(response.body, content_encoding)) self.assertEqual(data, params) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_query_params) d.addErrback(self.fail) @@ -517,7 +533,7 @@ class Https2ClientProtocolTestCase(TestCase): d_list = [] for status in [200, 404]: request = Request(self.get_url(f'/status?n={status}')) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_response_status, status) d.addErrback(self.fail) d_list.append(d) @@ -537,7 +553,7 @@ class Https2ClientProtocolTestCase(TestCase): self.assertIsInstance(response.ip_address, IPv4Address) self.assertEqual(str(response.ip_address), '127.0.0.1') - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_metadata) d.addErrback(self.fail) @@ -553,7 +569,7 @@ class Https2ClientProtocolTestCase(TestCase): self.assertIn('127.0.0.1', error_msg) self.assertIn(str(request), error_msg) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(self.fail) d.addErrback(assert_invalid_hostname) return d From 3685e99cca47a5006258cd4246f255216797641f Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 22 Jul 2020 14:47:20 +0530 Subject: [PATCH 034/114] test: http2 connection timeout --- tests/test_http2_client_protocol.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index d0386f7f8..746eef4d6 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -18,8 +18,9 @@ from twisted.web.client import URI from twisted.web.http import Request as TxRequest from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File +from twisted.internet.error import TimeoutError -from scrapy.core.http2.protocol import H2ClientFactory +from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.settings import Settings @@ -134,6 +135,11 @@ class NoContentLengthHeader(LeafResource): request.finish() +class TimeoutResponse(LeafResource): + def render_GET(self, request: TxRequest): + return NOT_DONE_YET + + class QueryParams(LeafResource): def render_GET(self, request: TxRequest): request.setHeader('Content-Type', 'application/json; charset=UTF-8') @@ -172,6 +178,7 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b'no-content-length-header', NoContentLengthHeader()) r.putChild(b'status', Status()) r.putChild(b'query-params', QueryParams()) + r.putChild(b'timeout', TimeoutResponse()) return r @inlineCallbacks @@ -590,3 +597,22 @@ class Https2ClientProtocolTestCase(TestCase): ] return DeferredList(d_list, fireOnOneErrback=True) + + def test_connection_timeout(self): + request = Request(self.get_url('/timeout')) + d = self.make_request(request) + + # Update the timer to 1s to test connection timeout + self.client.setTimeout(1) + + def assert_timeout_error(failure: Failure): + for err in failure.value.reasons: + if isinstance(err, TimeoutError): + self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err)) + break + else: + self.fail() + + d.addCallback(self.fail) + d.addErrback(assert_timeout_error) + return d From 9fffb801ed3dedbb3935c811c7d61ed953ff22dc Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 8 Jul 2020 20:18:38 +0530 Subject: [PATCH 035/114] feat: H2Agent, H2ConnectionPool base implementation --- scrapy/core/downloader/handlers/http2.py | 55 ++++++++++++++++++++ scrapy/core/http2/agent.py | 64 ++++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 scrapy/core/downloader/handlers/http2.py create mode 100644 scrapy/core/http2/agent.py diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py new file mode 100644 index 000000000..0dc06f4d8 --- /dev/null +++ b/scrapy/core/downloader/handlers/http2.py @@ -0,0 +1,55 @@ +import warnings + +from scrapy.core.downloader.tls import openssl_methods +from scrapy.core.http2.agent import H2Agent, H2ConnectionPool +from scrapy.http.request import Request +from scrapy.settings import Settings +from scrapy.utils.misc import create_instance, load_object + + +class H2DownloadHandler: + def __init__(self, settings: Settings, crawler=None): + self._crawler = crawler + + from twisted.internet import reactor + self._pool = H2ConnectionPool(reactor, settings) + + self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] + self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + # try method-aware context factory + try: + self._context_factory = create_instance( + objcls=self._context_factory_cls, + settings=settings, + crawler=crawler, + method=self._ssl_method, + ) + except TypeError: + # use context factory defaults + self._context_factory = create_instance( + objcls=self._context_factory_cls, + settings=settings, + crawler=crawler, + ) + msg = """ + '%s' does not accept `method` argument (type OpenSSL.SSL method,\ + e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ + Please upgrade your context factory class to handle them or ignore them.""" % ( + settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + warnings.warn(msg) + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler.settings, crawler) + + def download_request(self, request: Request, spider): + from twisted.internet import reactor + + agent = H2Agent(reactor, self._pool, self._context_factory) + d = agent.request(request) + + def print_result(result): + print(result) + + d.addCallback(print_result) + return d diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py new file mode 100644 index 000000000..e14e5d633 --- /dev/null +++ b/scrapy/core/http2/agent.py @@ -0,0 +1,64 @@ +from typing import Dict, Tuple + +from twisted.internet import defer +from twisted.internet.base import ReactorBase +from twisted.internet.defer import Deferred +from twisted.internet.endpoints import SSL4ClientEndpoint, optionsForClientTLS +from twisted.web.client import URI, BrowserLikePolicyForHTTPS + +from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory +from scrapy.http.request import Request +from scrapy.settings import Settings +from scrapy.utils.python import to_bytes, to_unicode + + +class H2ConnectionPool: + def __init__(self, reactor: ReactorBase, settings: Settings) -> None: + self._reactor = reactor + self.settings = settings + self._connections: Dict[Tuple, H2ClientProtocol] = {} + + def get_connection(self, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: + key = (uri.scheme, uri.host, uri.port) + conn = self._connections.get(key, None) + if conn: + return defer.succeed(conn) + return self._new_connection(key, uri, endpoint) + + def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: + factory = H2ClientFactory(uri, self.settings) + d = endpoint.connect(factory) + + def put_connection(conn: H2ClientProtocol) -> H2ClientProtocol: + self._connections[key] = conn + return conn + + d.addCallback(put_connection) + return d + + def _remove_connection(self, key) -> None: + conn = self._connections.pop(key) + conn.loseConnection() + + +class H2Agent: + def __init__( + self, reactor: ReactorBase, pool: H2ConnectionPool, + context_factory=BrowserLikePolicyForHTTPS() + ) -> None: + self._reactor = reactor + self._pool = pool + self._context_factory = context_factory + + def request(self, request: Request) -> Deferred: + uri = URI.fromBytes(to_bytes(request.url, encoding='ascii')) + # options = optionsForClientTLS(hostname=to_unicode(uri.host), acceptableProtocols=[b'h2']) + # Hacky fix: Use options instead of self._context_factory to make endpoint work for HTTP/2 + endpoint = SSL4ClientEndpoint(self._reactor, to_unicode(uri.host), uri.port, self._context_factory) + d = self._pool.get_connection(uri, endpoint) + + def cb_connected(conn: H2ClientProtocol): + return conn.request(request) + + d.addCallback(cb_connected) + return d From 8252a6f8d8930ce23bdf8a6f51038b4ce49d1968 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 15 Jul 2020 04:58:59 +0530 Subject: [PATCH 036/114] fix: H2Agent not able to connect via SSL - add H2WrappedContextFactory class which wraps the context factory passed to H2Agent and updates the SSL context acceptable protocols list to only h2 --- scrapy/core/downloader/handlers/http2.py | 1 + scrapy/core/http2/agent.py | 66 +++++++++++++++--------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 0dc06f4d8..c8b401e80 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -50,6 +50,7 @@ class H2DownloadHandler: def print_result(result): print(result) + return result d.addCallback(print_result) return d diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index e14e5d633..566c074c2 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,15 +1,19 @@ -from typing import Dict, Tuple +from typing import Dict, Tuple, Optional from twisted.internet import defer +from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred -from twisted.internet.endpoints import SSL4ClientEndpoint, optionsForClientTLS -from twisted.web.client import URI, BrowserLikePolicyForHTTPS +from twisted.internet.endpoints import SSL4ClientEndpoint +from twisted.python.failure import Failure +from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory +from twisted.web.iweb import IPolicyForHTTPS +from zope.interface import implementer +from zope.interface.verify import verifyObject from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory from scrapy.http.request import Request from scrapy.settings import Settings -from scrapy.utils.python import to_bytes, to_unicode class H2ConnectionPool: @@ -26,39 +30,51 @@ class H2ConnectionPool: return self._new_connection(key, uri, endpoint) def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: - factory = H2ClientFactory(uri, self.settings) + conn_lost_deferred = Deferred() + conn_lost_deferred.addCallback(self._remove_connection, key) + + factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) d = endpoint.connect(factory) - - def put_connection(conn: H2ClientProtocol) -> H2ClientProtocol: - self._connections[key] = conn - return conn - - d.addCallback(put_connection) + d.addCallback(self.put_connection, key) return d - def _remove_connection(self, key) -> None: - conn = self._connections.pop(key) - conn.loseConnection() + def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol: + self._connections[key] = conn + return conn + + def _remove_connection(self, reason: Failure, key: Tuple) -> None: + self._connections.pop(key) + + +@implementer(IPolicyForHTTPS) +class H2WrappedContextFactory: + def __init__(self, context_factory) -> None: + verifyObject(IPolicyForHTTPS, context_factory) + self._wrapped_context_factory = context_factory + + def creatorForNetloc(self, hostname, port) -> ClientTLSOptions: + options = self._wrapped_context_factory.creatorForNetloc(hostname, port) + _setAcceptableProtocols(options._ctx, [b'h2']) + return options class H2Agent: def __init__( self, reactor: ReactorBase, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS() + context_factory=BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None ) -> None: self._reactor = reactor self._pool = pool - self._context_factory = context_factory + self._context_factory = H2WrappedContextFactory(context_factory) + self._endpoint_factory = _StandardEndpointFactory( + self._reactor, self._context_factory, + connect_timeout, bind_address + ) def request(self, request: Request) -> Deferred: - uri = URI.fromBytes(to_bytes(request.url, encoding='ascii')) - # options = optionsForClientTLS(hostname=to_unicode(uri.host), acceptableProtocols=[b'h2']) - # Hacky fix: Use options instead of self._context_factory to make endpoint work for HTTP/2 - endpoint = SSL4ClientEndpoint(self._reactor, to_unicode(uri.host), uri.port, self._context_factory) + uri = URI.fromBytes(bytes(request.url, encoding='utf-8')) + endpoint = self._endpoint_factory.endpointForURI(uri) d = self._pool.get_connection(uri, endpoint) - - def cb_connected(conn: H2ClientProtocol): - return conn.request(request) - - d.addCallback(cb_connected) + d.addCallback(lambda conn: conn.request(request)) return d From 62ce842afc8ef829ffd6f164712a8a9413eb9e1d Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 15 Jul 2020 07:50:53 +0530 Subject: [PATCH 037/114] fix: multiple h2 connections to same uri - When multiple requests are sent to H2ConnectionPool to the same uri while the connection is in connecting state -- multiple connections were establised. - Fixed the bug using a deque of all the request deferred's which fire with the H2ClientProtocol (connection) instance when connection is established --- scrapy/core/http2/agent.py | 55 ++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 566c074c2..7a8847c38 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,11 +1,11 @@ -from typing import Dict, Tuple, Optional +from collections import deque +from typing import Deque, Dict, List, Tuple, Optional from twisted.internet import defer from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred -from twisted.internet.endpoints import SSL4ClientEndpoint -from twisted.python.failure import Failure +from twisted.internet.endpoints import HostnameEndpoint from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory from twisted.web.iweb import IPolicyForHTTPS from zope.interface import implementer @@ -20,31 +20,70 @@ class H2ConnectionPool: def __init__(self, reactor: ReactorBase, settings: Settings) -> None: self._reactor = reactor self.settings = settings + + # Store a dictionary which is used to get the respective + # H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port) self._connections: Dict[Tuple, H2ClientProtocol] = {} - def get_connection(self, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: + # Save all requests that arrive before the connection is established + self._pending_requests: Dict[Tuple, Deque[Deferred]] = {} + + def get_connection(self, uri: URI, endpoint: HostnameEndpoint) -> Deferred: key = (uri.scheme, uri.host, uri.port) + if key in self._pending_requests: + # Received a request while connecting to remote + # Create a deferred which will fire with the H2ClientProtocol + # instance + d = Deferred() + self._pending_requests[key].append(d) + return d + + # Check if we already have a connection to the remote conn = self._connections.get(key, None) if conn: + # Return this connection instance wrapped inside a deferred return defer.succeed(conn) + + # No connection is established for the given URI return self._new_connection(key, uri, endpoint) - def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: + def _new_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred: + self._pending_requests[key] = deque() + conn_lost_deferred = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) - d = endpoint.connect(factory) - d.addCallback(self.put_connection, key) + conn_d = endpoint.connect(factory) + conn_d.addCallback(self.put_connection, key) + + d = Deferred() + self._pending_requests[key].append(d) return d def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol: self._connections[key] = conn + + # Now as we have established a proper HTTP/2 connection + # we fire all the deferred's with the connection instance + pending_requests = self._pending_requests.pop(key) + while pending_requests: + d = pending_requests.popleft() + d.callback(conn) + + del pending_requests + return conn - def _remove_connection(self, reason: Failure, key: Tuple) -> None: + def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None: self._connections.pop(key) + # Call the errback of all the pending requests for this connection + pending_requests = self._pending_requests.pop(key, None) + while pending_requests: + d = pending_requests.popleft() + d.errback(errors) + @implementer(IPolicyForHTTPS) class H2WrappedContextFactory: From 031bfc9c3bed6c4dbfa4b7fcd48a8fc15f3582fb Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 22 Jul 2020 15:01:59 +0530 Subject: [PATCH 038/114] feat(wip): ScrapyH2Agent, ScrapyProxyH2Agent --- scrapy/core/downloader/contextfactory.py | 32 +++++ scrapy/core/downloader/handlers/http11.py | 27 +--- scrapy/core/downloader/handlers/http2.py | 161 +++++++++++++++++----- scrapy/core/http2/agent.py | 35 ++++- 4 files changed, 190 insertions(+), 65 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 452242d47..c0463cfc7 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,8 +1,12 @@ from OpenSSL import SSL +import warnings + from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer +from scrapy.core.downloader.tls import openssl_methods +from scrapy.utils.misc import create_instance, load_object from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS @@ -92,3 +96,31 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): trustRoot=platformTrust(), extraCertificateOptions={'method': self._ssl_method}, ) + + +def load_context_factory_from_settings(settings, crawler): + ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] + context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + # try method-aware context factory + try: + context_factory = create_instance( + objcls=context_factory_cls, + settings=settings, + crawler=crawler, + method=ssl_method, + ) + except TypeError: + # use context factory defaults + context_factory = create_instance( + objcls=context_factory_cls, + settings=settings, + crawler=crawler, + ) + msg = """ + '%s' does not accept `method` argument (type OpenSSL.SSL method,\ + e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ + Please upgrade your context factory class to handle them or ignore them.""" % ( + settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + warnings.warn(msg) + + return context_factory diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 22c9ac520..dac97ad29 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -20,12 +20,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from zope.interface import implementer from scrapy import signals -from scrapy.core.downloader.tls import openssl_methods +from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload from scrapy.http import Headers from scrapy.responsetypes import responsetypes -from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import to_bytes, to_unicode @@ -43,29 +42,7 @@ class HTTP11DownloadHandler: self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False - self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] - self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - # try method-aware context factory - try: - self._contextFactory = create_instance( - objcls=self._contextFactoryClass, - settings=settings, - crawler=crawler, - method=self._sslMethod, - ) - except TypeError: - # use context factory defaults - self._contextFactory = create_instance( - objcls=self._contextFactoryClass, - settings=settings, - crawler=crawler, - ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) - warnings.warn(msg) + self._contextFactory = load_context_factory_from_settings(settings, crawler) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index c8b401e80..81ea78e69 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,10 +1,19 @@ import warnings +from time import time +from typing import Optional, Tuple +from urllib.parse import urldefrag -from scrapy.core.downloader.tls import openssl_methods +from twisted.internet.base import ReactorBase +from twisted.internet.error import TimeoutError +from twisted.web.client import URI + +from scrapy.core.downloader.contextfactory import load_context_factory_from_settings +from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool -from scrapy.http.request import Request +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.http import Request, Response from scrapy.settings import Settings -from scrapy.utils.misc import create_instance, load_object +from scrapy.spiders import Spider class H2DownloadHandler: @@ -13,44 +22,128 @@ class H2DownloadHandler: from twisted.internet import reactor self._pool = H2ConnectionPool(reactor, settings) - - self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] - self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - # try method-aware context factory - try: - self._context_factory = create_instance( - objcls=self._context_factory_cls, - settings=settings, - crawler=crawler, - method=self._ssl_method, - ) - except TypeError: - # use context factory defaults - self._context_factory = create_instance( - objcls=self._context_factory_cls, - settings=settings, - crawler=crawler, - ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) - warnings.warn(msg) + self._context_factory = load_context_factory_from_settings(settings, crawler) + self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') + self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') + self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') @classmethod def from_crawler(cls, crawler): return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider): + def download_request(self, request: Request, spider: Spider): + agent = ScrapyH2Agent( + context_factory=self._context_factory, + pool=self._pool, + maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), + warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), + crawler=self._crawler + ) + return agent.download_request(request, spider) + + def close(self) -> None: + self._pool.close_connections() + + +class ScrapyProxyH2Agent(H2Agent): + def __init__( + self, reactor: ReactorBase, + proxy_uri: URI, pool: H2ConnectionPool, + connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + ) -> None: + super(ScrapyProxyH2Agent, self).__init__( + reactor=reactor, + pool=pool, + connect_timeout=connect_timeout, + bind_address=bind_address + ) + self._proxy_uri = proxy_uri + + @staticmethod + def get_key(uri: URI) -> Tuple: + return "http-proxy", uri.host, uri.port + + +class ScrapyH2Agent: + _Agent = H2Agent + _ProxyAgent = ScrapyProxyH2Agent + + def __init__( + self, context_factory, + connect_timeout=10, + bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None, + maxsize: int = 0, warnsize: int = 0, + crawler=None + ) -> None: + self._context_factory = context_factory + self._connect_timeout = connect_timeout + self._bind_address = bind_address + self._pool = pool + self._maxsize = maxsize + self._warnsize = warnsize + self._crawler = crawler + + def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent: from twisted.internet import reactor + bind_address = request.meta.get('bindaddress') or self._bind_address + proxy = request.meta.get('proxy') + if proxy: + _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) + scheme = _parse(request.url)[0] + proxy_host = str(proxy_host, 'utf-8') + omit_connect_timeout = b'noconnect' in proxy_params + if omit_connect_timeout: + warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " + "If you use Crawlera, it doesn't require this mode anymore, " + "so you should update scrapy-crawlera to 1.3.0+ " + "and remove '?noconnect' from the Crawlera URL.", + ScrapyDeprecationWarning) - agent = H2Agent(reactor, self._pool, self._context_factory) - d = agent.request(request) + if scheme == b'https' and not omit_connect_timeout: + proxy_auth = request.headers.get(b'Proxy-Authorization', None) + proxy_conf = (proxy_host, proxy_port, proxy_auth) - def print_result(result): - print(result) - return result + # TODO: Return TunnelingAgent instance + else: + return self._ProxyAgent( + reactor=reactor, + proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')), + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool + ) - d.addCallback(print_result) + return self._Agent( + reactor=reactor, + context_factory=self._context_factory, + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool + ) + + def download_request(self, request: Request, spider: Spider): + from twisted.internet import reactor + timeout = request.meta.get('download_timeout') or self._connect_timeout + agent = self._get_agent(request, timeout) + + start_time = time() + d = agent.request(request, spider) + d.addCallback(self._cb_latency, request, start_time) + + timeout_cl = reactor.callLater(timeout, d.cancel) + d.addBoth(self._cb_timeout, request, timeout, timeout_cl) return d + + @staticmethod + def _cb_latency(response: Response, request: Request, start_time: float): + request.meta['download_latency'] = time() - start_time + return response + + @staticmethod + def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl): + if timeout_cl.active(): + timeout_cl.cancel() + return response + + url = urldefrag(request.url)[0] + raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 7a8847c38..c7a49fd42 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -6,7 +6,9 @@ from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOption from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.endpoints import HostnameEndpoint +from twisted.python.failure import Failure from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory +from twisted.web.error import SchemeNotSupported from twisted.web.iweb import IPolicyForHTTPS from zope.interface import implementer from zope.interface.verify import verifyObject @@ -14,6 +16,7 @@ from zope.interface.verify import verifyObject from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory from scrapy.http.request import Request from scrapy.settings import Settings +from scrapy.spiders import Spider class H2ConnectionPool: @@ -28,8 +31,7 @@ class H2ConnectionPool: # Save all requests that arrive before the connection is established self._pending_requests: Dict[Tuple, Deque[Deferred]] = {} - def get_connection(self, uri: URI, endpoint: HostnameEndpoint) -> Deferred: - key = (uri.scheme, uri.host, uri.port) + def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred: if key in self._pending_requests: # Received a request while connecting to remote # Create a deferred which will fire with the H2ClientProtocol @@ -84,6 +86,15 @@ class H2ConnectionPool: d = pending_requests.popleft() d.errback(errors) + def close_connections(self) -> None: + """Close all the HTTP/2 connections and remove them from pool + + Returns: + Deferred that fires when all connections have been closed + """ + for conn in self._connections.values(): + conn.transport.loseConnection() + @implementer(IPolicyForHTTPS) class H2WrappedContextFactory: @@ -111,9 +122,21 @@ class H2Agent: connect_timeout, bind_address ) - def request(self, request: Request) -> Deferred: + def _get_endpoint(self, uri: URI): + return self._endpoint_factory.endpointForURI(uri) + + @staticmethod + def get_key(uri: URI) -> Tuple: + return uri.scheme, uri.host, uri.port + + def request(self, request: Request, spider: Spider) -> Deferred: uri = URI.fromBytes(bytes(request.url, encoding='utf-8')) - endpoint = self._endpoint_factory.endpointForURI(uri) - d = self._pool.get_connection(uri, endpoint) - d.addCallback(lambda conn: conn.request(request)) + try: + endpoint = self._get_endpoint(uri) + except SchemeNotSupported: + return defer.fail(Failure()) + + key = self.get_key(uri) + d = self._pool.get_connection(key, uri, endpoint) + d.addCallback(lambda conn: conn.request(request, spider)) return d From 92bec38591fccb523c2e643aef70a1f6cd7267ea Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 29 Jul 2020 13:43:59 +0530 Subject: [PATCH 039/114] feat: MethodNotAllowed405, Content-Length header - add tests to check for Content-Length header - raise MethodNotAllowed405 when remote send 'HTTP/2.0 405 Method Not Allowed' --- scrapy/core/downloader/handlers/http2.py | 15 +++----- scrapy/core/http2/agent.py | 2 +- scrapy/core/http2/protocol.py | 32 +++++++++++++---- scrapy/core/http2/stream.py | 14 ++++++-- tests/test_http2_client_protocol.py | 46 ++++++++++++++++++++++-- 5 files changed, 85 insertions(+), 24 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 81ea78e69..e9cc5ebbc 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -3,6 +3,7 @@ from time import time from typing import Optional, Tuple from urllib.parse import urldefrag +from twisted.internet.defer import Deferred from twisted.internet.base import ReactorBase from twisted.internet.error import TimeoutError from twisted.web.client import URI @@ -23,9 +24,6 @@ class H2DownloadHandler: from twisted.internet import reactor self._pool = H2ConnectionPool(reactor, settings) self._context_factory = load_context_factory_from_settings(settings, crawler) - self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') - self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') - self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') @classmethod def from_crawler(cls, crawler): @@ -35,8 +33,6 @@ class H2DownloadHandler: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, - maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), - warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), crawler=self._crawler ) return agent.download_request(request, spider) @@ -72,15 +68,12 @@ class ScrapyH2Agent: self, context_factory, connect_timeout=10, bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None, - maxsize: int = 0, warnsize: int = 0, crawler=None ) -> None: self._context_factory = context_factory self._connect_timeout = connect_timeout self._bind_address = bind_address self._pool = pool - self._maxsize = maxsize - self._warnsize = warnsize self._crawler = crawler def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent: @@ -121,7 +114,7 @@ class ScrapyH2Agent: pool=self._pool ) - def download_request(self, request: Request, spider: Spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: from twisted.internet import reactor timeout = request.meta.get('download_timeout') or self._connect_timeout agent = self._get_agent(request, timeout) @@ -135,12 +128,12 @@ class ScrapyH2Agent: return d @staticmethod - def _cb_latency(response: Response, request: Request, start_time: float): + def _cb_latency(response: Response, request: Request, start_time: float) -> Response: request.meta['download_latency'] = time() - start_time return response @staticmethod - def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl): + def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl) -> Response: if timeout_cl.active(): timeout_cl.cancel() return response diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index c7a49fd42..e62eef263 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -93,7 +93,7 @@ class H2ConnectionPool: Deferred that fires when all connections have been closed """ for conn in self._connections.values(): - conn.transport.loseConnection() + conn.transport.abortConnection() @implementer(IPolicyForHTTPS) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index feb034a0c..1ce8b6548 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -13,7 +13,7 @@ from h2.events import ( SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived, WindowUpdated ) -from h2.exceptions import H2Error, ProtocolError +from h2.exceptions import H2Error from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory @@ -39,7 +39,7 @@ class InvalidNegotiatedProtocol(H2Error): self.negotiated_protocol = negotiated_protocol def __str__(self) -> str: - return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol}' + return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}' class RemoteTerminatedConnection(H2Error): @@ -48,7 +48,15 @@ class RemoteTerminatedConnection(H2Error): self.terminate_event = event def __str__(self) -> str: - return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address}' + return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address!r}' + + +class MethodNotAllowed405(H2Error): + def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]): + self.remote_ip_address = remote_ip_address + + def __str__(self) -> str: + return f"MethodNotAllowed405: Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}" @implementer(IHandshakeListener) @@ -217,14 +225,25 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # So, no need to send a GOAWAY frame to the remote self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) + def _check_received_data(self, data: bytes) -> None: + """Checks for edge cases where the connection to remote fails + without raising an appropriate H2Error + + Arguments: + data -- Data received from the remote + """ + if data.startswith(b'HTTP/2.0 405 Method Not Allowed'): + raise MethodNotAllowed405(self.metadata['ip_address']) + def dataReceived(self, data: bytes) -> None: # Reset the idle timeout as connection is still actively receiving data self.resetTimeout() try: + self._check_received_data(data) events = self.conn.receive_data(data) self._handle_events(events) - except ProtocolError as e: + except H2Error as e: # Save this error as ultimately the connection will be dropped # internally by hyper-h2. Saved error will be passed to all the streams # closed with the connection. @@ -271,9 +290,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): for stream in self.streams.values(): if stream.request_sent: - stream.close(StreamCloseReason.CONNECTION_LOST, self._conn_lost_errors, from_protocol=True) + close_reason = StreamCloseReason.CONNECTION_LOST else: - stream.close(StreamCloseReason.INACTIVE, from_protocol=True) + close_reason = StreamCloseReason.INACTIVE + stream.close(close_reason, self._conn_lost_errors, from_protocol=True) self._active_streams -= len(self.streams) self.streams.clear() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 133196797..5bffa67e7 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -189,12 +189,15 @@ class Stream: headers = [ (':method', self._request.method), (':authority', url.netloc), - (':scheme', 'https'), + (':scheme', self._protocol.metadata['uri'].scheme), (':path', path), ] for name, value in self._request.headers.items(): - headers.append((name, value[0])) + headers.append((str(name, 'utf-8'), str(value[0], 'utf-8'))) + + if b'Content-Length' not in self._request.headers.keys(): + headers.append(('Content-Length', str(len(self._request.body)))) return headers @@ -337,6 +340,10 @@ class Stream: if not isinstance(reason, StreamCloseReason): raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}') + # Have default value of errors as an empty list as + # some cases can add a list of exceptions + errors = errors or [] + if not from_protocol: self._protocol.pop_stream(self.stream_id) @@ -387,7 +394,8 @@ class Stream: self._deferred_response.errback(ResponseFailed(errors)) elif reason is StreamCloseReason.INACTIVE: - self._deferred_response.errback(InactiveStreamClosed(self._request)) + errors.insert(0, InactiveStreamClosed(self._request)) + self._deferred_response.errback(ResponseFailed(errors)) elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 746eef4d6..4926ada14 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -11,14 +11,14 @@ from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint +from twisted.internet.error import TimeoutError from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase -from twisted.web.client import URI +from twisted.web.client import ResponseFailed, URI from twisted.web.http import Request as TxRequest from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File -from twisted.internet.error import TimeoutError from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname @@ -152,6 +152,19 @@ class QueryParams(LeafResource): return bytes(json.dumps(query_params), 'utf-8') +class RequestHeaders(LeafResource): + """Sends all the headers received as a response""" + + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') + headers = {} + for k, v in request.requestHeaders.getAllRawHeaders(): + headers[str(k, 'utf-8')] = str(v[0], 'utf-8') + + return bytes(json.dumps(headers), 'utf-8') + + def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) @@ -179,6 +192,7 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b'status', Status()) r.putChild(b'query-params', QueryParams()) r.putChild(b'timeout', TimeoutResponse()) + r.putChild(b'request-headers', RequestHeaders()) return r @inlineCallbacks @@ -488,7 +502,11 @@ class Https2ClientProtocolTestCase(TestCase): d_list = [] def assert_inactive_stream(failure): - self.assertIsNotNone(failure.check(InactiveStreamClosed)) + self.assertIsNotNone(failure.check(ResponseFailed)) + self.assertTrue(any( + isinstance(e, InactiveStreamClosed) + for e in failure.value.reasons + )) # Send 100 request (we do not check the result) for _ in range(100): @@ -616,3 +634,25 @@ class Https2ClientProtocolTestCase(TestCase): d.addCallback(self.fail) d.addErrback(assert_timeout_error) return d + + def test_request_headers_received(self): + request = Request(self.get_url('/request-headers'), headers={ + 'header-1': 'header value 1', + 'header-2': 'header value 2' + }) + d = self.make_request(request) + + def assert_request_headers(response: Response): + self.assertEqual(response.status, 200) + self.assertEqual(response.request, request) + + response_headers = json.loads(str(response.body, 'utf-8')) + self.assertIsInstance(response_headers, dict) + for k, v in request.headers.items(): + k, v = str(k, 'utf-8'), str(v[0], 'utf-8') + self.assertIn(k, response_headers) + self.assertEqual(v, response_headers[k]) + + d.addErrback(self.fail) + d.addCallback(assert_request_headers) + return d From e8342996f6273b6b60ffc26a67e90dec34e96dfd Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 29 Jul 2020 13:51:01 +0530 Subject: [PATCH 040/114] test: H2DownloadHandler Following tests are skipped as Content-Length header not matching the data received is considered as a ProtocolError - test_download_broken_content_cause_data_loss - test_download_broken_chunked_content_cause_data_loss - test_download_broken_content_allow_data_loss - test_download_broken_chunked_content_allow_data_loss - test_download_broken_content_allow_data_loss_via_setting - test_download_broken_chunked_content_allow_data_loss_via_setting BREAKING CHANGES The following tests currently fail - test_content_length_zero_bodyless_post_request_headers - test_host_header_seted_in_request_headers - test_download_with_maxsize_very_large_file --- tests/test_downloader_handlers.py | 108 +++++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 9 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 51deb20f4..f6add82dc 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -23,8 +23,8 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler +from scrapy.core.downloader.handlers.http2 import H2DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler - from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Headers, Request from scrapy.http.response.text import TextResponse @@ -33,7 +33,6 @@ from scrapy.spiders import Spider from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto - from tests.mockserver import MockServer, ssl_context_factory, Echo from tests.spiders import SingleRequestSpider @@ -131,6 +130,7 @@ class ContentLengthHeaderResource(resource.Resource): A testing resource which renders itself as the value of the Content-Length header from the request. """ + def render(self, request): return request.requestHeaders.getRawHeaders(b"content-length")[0] @@ -142,6 +142,7 @@ class ChunkedResource(resource.Resource): request.write(b"chunked ") request.write(b"content\n") request.finish() + reactor.callLater(0, response) return server.NOT_DONE_YET @@ -155,6 +156,7 @@ class BrokenChunkedResource(resource.Resource): # Disable terminating chunk on finish. request.chunked = False closeConnection(request) + reactor.callLater(0, response) return server.NOT_DONE_YET @@ -186,6 +188,7 @@ class EmptyContentTypeHeaderResource(resource.Resource): A testing resource which renders itself as the value of request body without content-type header in response. """ + def render(self, request): request.setHeader("content-type", "") return request.content.read() @@ -197,12 +200,12 @@ class LargeChunkedFileResource(resource.Resource): for i in range(1024): request.write(b"x" * 1024) request.finish() + reactor.callLater(0, response) return server.NOT_DONE_YET class HttpTestCase(unittest.TestCase): - scheme = 'http' download_handler_cls = HTTPDownloadHandler @@ -230,10 +233,12 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"echo", Echo()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) - self.host = 'localhost' + self.host = u'localhost' if self.scheme == 'https': + # Using WrappingFactory do not enable HTTP/2 failing all the + # tests with H2DownloadHandler self.port = reactor.listenSSL( - 0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile), + 0, self.site, ssl_context_factory(self.keyfile, self.certfile), interface=self.host) else: self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) @@ -330,6 +335,7 @@ class HttpTestCase(unittest.TestCase): https://github.com/kennethreitz/requests/issues/405 https://bugs.python.org/issue14721 """ + def _test(response): self.assertEqual(response.body, b'0') @@ -514,6 +520,30 @@ class Https11TestCase(Http11TestCase): yield download_handler.close() +class Https2TestCase(Https11TestCase): + scheme = 'https' + download_handler_cls = H2DownloadHandler + HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + + def test_download_broken_content_cause_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_cause_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss_via_setting(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + class Https11WrongHostnameTestCase(Http11TestCase): scheme = 'https' @@ -526,6 +556,23 @@ class Https11WrongHostnameTestCase(Http11TestCase): certfile = 'keys/example-com.cert.pem' +class Https2WrongHostnameTestCase(Https2TestCase): + tls_log_message = ( + 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' + 'subject "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' + ) + + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = 'keys/example-com.key.pem' + certfile = 'keys/example-com.cert.pem' + + class Https11InvalidDNSId(Https11TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" @@ -534,6 +581,14 @@ class Https11InvalidDNSId(Https11TestCase): self.host = '127.0.0.1' +class Https2InvalidDNSId(Https2TestCase): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super(Https2InvalidDNSId, self).setUp() + self.host = '127.0.0.1' + + class Https11InvalidDNSPattern(Https11TestCase): """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" @@ -552,6 +607,24 @@ class Https11InvalidDNSPattern(Https11TestCase): super(Https11InvalidDNSPattern, self).setUp() +class Https2InvalidDNSPattern(Https2TestCase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = 'keys/localhost.ip.key' + certfile = 'keys/localhost.ip.crt' + + def setUp(self): + try: + from service_identity.exceptions import CertificateError # noqa: F401 + except ImportError: + raise unittest.SkipTest("cryptography lib is too old") + self.tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' + 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' + ) + super(Https2InvalidDNSPattern, self).setUp() + + class Https11CustomCiphers(unittest.TestCase): scheme = 'https' download_handler_cls = HTTP11DownloadHandler @@ -565,10 +638,9 @@ class Https11CustomCiphers(unittest.TestCase): FilePath(self.tmpname).child("file").setContent(b"0123456789") r = static.File(self.tmpname) self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) self.host = 'localhost' self.port = reactor.listenSSL( - 0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'), + 0, self.site, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'), interface=self.host) self.portno = self.port.getHost().port crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}) @@ -593,6 +665,11 @@ class Https11CustomCiphers(unittest.TestCase): return d +class Https2CustomCiphers(Https11CustomCiphers): + scheme = 'https' + download_handler_cls = H2DownloadHandler + + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" @@ -644,6 +721,10 @@ class Http11MockServerTestCase(unittest.TestCase): self.assertTrue(reason, 'finished') +class Http2MockServerTestCase(Http11MockServerTestCase): + """HTTP 2.0 test case with MockServer""" + + class UriResource(resource.Resource): """Return the full uri that was requested""" @@ -734,6 +815,11 @@ class Http11ProxyTestCase(HttpProxyTestCase): self.assertIn(domain, timeout.osError) +# TODO: +class Http2ProxyTestCase(Http11ProxyTestCase): + download_handler_cls = H2DownloadHandler + + class HttpDownloadHandlerMock: def __init__(self, *args, **kwargs): @@ -931,7 +1017,6 @@ class S3TestCase(unittest.TestCase): class BaseFTPTestCase(unittest.TestCase): - username = "scrapy" password = "passwd" req_meta = {"ftp_user": username, "ftp_password": password} @@ -969,6 +1054,7 @@ class BaseFTPTestCase(unittest.TestCase): def _clean(data): self.download_handler.client.transport.loseConnection() return data + deferred.addCallback(_clean) if callback: deferred.addCallback(callback) @@ -985,6 +1071,7 @@ class BaseFTPTestCase(unittest.TestCase): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'I have the power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) + return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): @@ -998,6 +1085,7 @@ class BaseFTPTestCase(unittest.TestCase): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'Moooooooooo power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']}) + return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): @@ -1007,6 +1095,7 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.status, 404) + return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): @@ -1027,6 +1116,7 @@ class BaseFTPTestCase(unittest.TestCase): with open(local_fname, "rb") as f: self.assertEqual(f.read(), b"I have the power!") os.remove(local_fname) + return self._add_test_callbacks(d, _test) @@ -1043,11 +1133,11 @@ class FTPTestCase(BaseFTPTestCase): def _test(r): self.assertEqual(r.type, ConnectionLost) + return self._add_test_callbacks(d, errback=_test) class AnonymousFTPTestCase(BaseFTPTestCase): - username = "anonymous" req_meta = {} From 19f2b4b53dd51044083a9749f00366f41ed795c7 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 29 Jul 2020 17:25:59 +0530 Subject: [PATCH 041/114] refactor: AcceptableProtocolsContextFactory - rename H2WrappedContextFactory to AcceptableProtocolsContextFactory - AcceptableProtocolsContextFactory accepts an argument acceptable_protocols which can be used to override the context factory priority list of protocols during ALPN or NPN --- scrapy/core/downloader/handlers/http2.py | 2 +- scrapy/core/http2/agent.py | 9 +++++---- scrapy/core/http2/stream.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index e9cc5ebbc..411e06a78 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -3,8 +3,8 @@ from time import time from typing import Optional, Tuple from urllib.parse import urldefrag -from twisted.internet.defer import Deferred from twisted.internet.base import ReactorBase +from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index e62eef263..aa51508a5 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -97,14 +97,15 @@ class H2ConnectionPool: @implementer(IPolicyForHTTPS) -class H2WrappedContextFactory: - def __init__(self, context_factory) -> None: +class AcceptableProtocolsContextFactory: + def __init__(self, context_factory, acceptable_protocols: List[bytes]) -> None: verifyObject(IPolicyForHTTPS, context_factory) self._wrapped_context_factory = context_factory + self._acceptable_protocols = acceptable_protocols def creatorForNetloc(self, hostname, port) -> ClientTLSOptions: options = self._wrapped_context_factory.creatorForNetloc(hostname, port) - _setAcceptableProtocols(options._ctx, [b'h2']) + _setAcceptableProtocols(options._ctx, self._acceptable_protocols) return options @@ -116,7 +117,7 @@ class H2Agent: ) -> None: self._reactor = reactor self._pool = pool - self._context_factory = H2WrappedContextFactory(context_factory) + self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2']) self._endpoint_factory = _StandardEndpointFactory( self._reactor, self._context_factory, connect_timeout, bind_address diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 5bffa67e7..acdd46320 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -85,7 +85,7 @@ class Stream: request: Request, protocol: "H2ClientProtocol", download_maxsize: int = 0, - download_warnsize: int = 0 + download_warnsize: int = 0, ) -> None: """ Arguments: From a3fecaf07f9edd6a1bbac1ade825c23845c7e6b1 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 30 Jul 2020 15:45:27 +0530 Subject: [PATCH 042/114] test: fix host-name H2DownloadHandler tests --- tests/test_downloader_handlers.py | 33 +++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index f6add82dc..4dd32b6f5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -312,16 +312,18 @@ class HttpTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): - def _test(response): - self.assertEqual(response.body, b'example.com') - self.assertEqual(request.headers.get('Host'), b'example.com') + host = self.host + ':' + str(self.portno) - request = Request(self.getURL('host'), headers={'Host': 'example.com'}) + def _test(response): + self.assertEqual(response.body, bytes(host, 'utf-8')) + self.assertEqual(request.headers.get('Host'), bytes(host, 'utf-8')) + + request = Request(self.getURL('host'), headers={'Host': host}) return self.download_request(request, Spider('foo')).addCallback(_test) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b'example.com') + d.addCallback(self.assertEqual, b'localhost') return d def test_content_length_zero_bodyless_post_request_headers(self): @@ -339,7 +341,7 @@ class HttpTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.body, b'0') - request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'}) + request = Request(self.getURL('contentlength'), method='POST') return self.download_request(request, Spider('foo')).addCallback(_test) def test_content_length_zero_bodyless_post_only_one(self): @@ -525,6 +527,25 @@ class Https2TestCase(Https11TestCase): download_handler_cls = H2DownloadHandler HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + @defer.inlineCallbacks + def test_download_with_maxsize_very_large_file(self): + with mock.patch('scrapy.core.http2.stream.logger') as logger: + request = Request(self.getURL('largechunkedfile')) + + def check(logger): + logger.error.assert_called_once_with(mock.ANY) + + d = self.download_request(request, Spider('foo', download_maxsize=1500)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + # As the error message is logged in the dataReceived callback, we + # have to give a bit of time to the reactor to process the queue + # after closing the connection. + d = defer.Deferred() + d.addCallback(check) + reactor.callLater(.1, d.callback, logger) + yield d + def test_download_broken_content_cause_data_loss(self, url='broken'): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) From d707f8b5d94856ba13bd8e76acb1efb8de377f9c Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 30 Jul 2020 18:06:21 +0530 Subject: [PATCH 043/114] docs: mention H2DownloadHandler in settings.rst --- docs/topics/settings.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5178f272f..670b44f3c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -620,6 +620,13 @@ handler (without replacement), place this in your ``settings.py``:: 'ftp': None, } +The default https handler uses HTTP/1.x, to use HTTP/2.0 update :setting:`DOWNLOAD_HANDLERS` +as:: + + DOWNLOAD_HANDLERS = { + 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', + } + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT @@ -697,6 +704,14 @@ Optionally, this can be set per-request basis by using the If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``, the ``ResponseFailed([_DataLoss])`` failure will be retried as usual. +.. warning:: + + This is ignored when :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` + is set as ``https`` download handler in :setting:`DOWNLOAD_HANDLERS`. In + case of data loss error the connection may be corrupted affecting other streams, + hence all streams return with the ``ResponseFailed([InvalidBodyLengthError])`` + failure. + .. setting:: DUPEFILTER_CLASS DUPEFILTER_CLASS From e0c3019d90f187482cd84b946b83c496411ec34b Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 9 Aug 2020 16:19:35 +0530 Subject: [PATCH 044/114] fix: ScrapyProxyH2Agent - add required test cases BREAKING CHANGES Presently the tests (in test_downloader_handlers.py) 1. test_download_without_proxy 2. test_download_with_proxy_https_timeout collide with each other when run together. However, if both of the tests are ran individually then both pass. --- scrapy/core/downloader/handlers/http2.py | 14 ++++++--- scrapy/core/http2/agent.py | 15 ++++++---- scrapy/core/http2/stream.py | 22 +++++++++++++-- tests/test_downloader_handlers.py | 36 ++++++++++++++++++++---- 4 files changed, 70 insertions(+), 17 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 411e06a78..d6e5cd1c1 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -6,7 +6,7 @@ from urllib.parse import urldefrag from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.web.client import URI +from twisted.web.client import URI, BrowserLikePolicyForHTTPS from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse @@ -45,19 +45,24 @@ class ScrapyProxyH2Agent(H2Agent): def __init__( self, reactor: ReactorBase, proxy_uri: URI, pool: H2ConnectionPool, + context_factory=BrowserLikePolicyForHTTPS(), connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None ) -> None: super(ScrapyProxyH2Agent, self).__init__( reactor=reactor, pool=pool, + context_factory=context_factory, connect_timeout=connect_timeout, bind_address=bind_address ) self._proxy_uri = proxy_uri - @staticmethod - def get_key(uri: URI) -> Tuple: - return "http-proxy", uri.host, uri.port + def get_endpoint(self, uri: URI): + return self.endpoint_factory.endpointForURI(self._proxy_uri) + + def get_key(self, uri: URI) -> Tuple: + """We use the proxy uri instead of uri obtained from request url""" + return "http-proxy", self._proxy_uri.host, self._proxy_uri.port class ScrapyH2Agent: @@ -100,6 +105,7 @@ class ScrapyH2Agent: else: return self._ProxyAgent( reactor=reactor, + context_factory=self._context_factory, proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')), connect_timeout=timeout, bind_address=bind_address, diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index aa51508a5..f4ac29bc6 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -118,22 +118,25 @@ class H2Agent: self._reactor = reactor self._pool = pool self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2']) - self._endpoint_factory = _StandardEndpointFactory( + self.endpoint_factory = _StandardEndpointFactory( self._reactor, self._context_factory, connect_timeout, bind_address ) - def _get_endpoint(self, uri: URI): - return self._endpoint_factory.endpointForURI(uri) + def get_endpoint(self, uri: URI): + return self.endpoint_factory.endpointForURI(uri) - @staticmethod - def get_key(uri: URI) -> Tuple: + def get_key(self, uri: URI) -> Tuple: + """ + Arguments: + uri - URI obtained directly from request URL + """ return uri.scheme, uri.host, uri.port def request(self, request: Request, spider: Spider) -> Deferred: uri = URI.fromBytes(bytes(request.url, encoding='utf-8')) try: - endpoint = self._get_endpoint(uri) + endpoint = self.get_endpoint(uri) except SchemeNotSupported: return defer.fail(Failure()) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index acdd46320..40ea07b63 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -185,14 +185,32 @@ class Stream: if url.query: path += '?' + url.query + # This pseudo-header field MUST NOT be empty for "http" or "https" + # URIs; "http" or "https" URIs that do not contain a path component + # MUST include a value of '/'. The exception to this rule is an + # OPTIONS request for an "http" or "https" URI that does not include + # a path component; these MUST include a ":path" pseudo-header field + # with a value of '*' (refer RFC 7540 - Section 8.1.2.3) + if not path: + if self._request.method == 'OPTIONS': + path = path or '*' + else: + path = path or '/' + # Make sure pseudo-headers comes before all the other headers headers = [ (':method', self._request.method), (':authority', url.netloc), - (':scheme', self._protocol.metadata['uri'].scheme), - (':path', path), ] + # The ":scheme" and ":path" pseudo-header fields MUST + # be omitted for CONNECT method (refer RFC 7540 - Section 8.3) + if self._request.method != 'CONNECT': + headers += [ + (':scheme', self._protocol.metadata['uri'].scheme), + (':path', path), + ] + for name, value in self._request.headers.items(): headers.append((str(name, 'utf-8'), str(value[0], 'utf-8'))) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 4dd32b6f5..486614121 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -786,7 +786,10 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'http://example.com') + self.assertTrue( + response.body == b'http://example.com' # HTTP/1.x + or response.body == b'/' # HTTP/2 + ) http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) @@ -796,10 +799,13 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'https://example.com') + self.assertTrue( + response.body == b'http://example.com' # HTTP/1.x + or response.body == b'/' # HTTP/2 + ) http_proxy = '%s?noconnect' % self.getURL('') - request = Request('https://example.com', meta={'proxy': http_proxy}) + request = Request('http://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex(ScrapyDeprecationWarning, r'Using HTTPS proxies in the noconnect mode is deprecated'): return self.download_request(request, Spider('foo')).addCallback(_test) @@ -836,10 +842,30 @@ class Http11ProxyTestCase(HttpProxyTestCase): self.assertIn(domain, timeout.osError) -# TODO: -class Http2ProxyTestCase(Http11ProxyTestCase): +class Https2ProxyTestCase(Http11ProxyTestCase): + # only used for HTTPS tests + keyfile = 'keys/localhost.key' + certfile = 'keys/localhost.crt' + + scheme = 'https' + host = u'127.0.0.1' + download_handler_cls = H2DownloadHandler + def setUp(self): + site = server.Site(UriResource(), timeout=None) + self.port = reactor.listenSSL( + 0, site, + ssl_context_factory(self.keyfile, self.certfile), + interface=self.host + ) + self.portno = self.port.getHost().port + self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) + self.download_request = self.download_handler.download_request + + def getURL(self, path): + return f"{self.scheme}://{self.host}:{self.portno}/{path}" + class HttpDownloadHandlerMock: From c67d6dea318d6f0915ae86d46ce367b6c1e8ee51 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 11 Aug 2020 04:39:41 +0530 Subject: [PATCH 045/114] fix: H2 docs, NotImplementedError for H2 Tunnel --- docs/topics/settings.rst | 15 ++++---- scrapy/core/downloader/contextfactory.py | 7 ++-- scrapy/core/downloader/handlers/http2.py | 44 +++++++++++------------- scrapy/core/http2/stream.py | 5 +-- tests/test_downloader_handlers.py | 33 ++++++++++++------ 5 files changed, 54 insertions(+), 50 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 670b44f3c..bb543433c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -620,8 +620,8 @@ handler (without replacement), place this in your ``settings.py``:: 'ftp': None, } -The default https handler uses HTTP/1.x, to use HTTP/2.0 update :setting:`DOWNLOAD_HANDLERS` -as:: +The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update +:setting:`DOWNLOAD_HANDLERS` as follows:: DOWNLOAD_HANDLERS = { 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', @@ -706,11 +706,12 @@ Optionally, this can be set per-request basis by using the .. warning:: - This is ignored when :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` - is set as ``https`` download handler in :setting:`DOWNLOAD_HANDLERS`. In - case of data loss error the connection may be corrupted affecting other streams, - hence all streams return with the ``ResponseFailed([InvalidBodyLengthError])`` - failure. + This setting is ignored by the + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` + download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss + error, the corresponding HTTP/2 connection may be corrupted, affecting other + requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])`` + failure is always raised for every request that was using that connection. .. setting:: DUPEFILTER_CLASS diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index c0463cfc7..8dcba15ff 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,14 +1,13 @@ -from OpenSSL import SSL import warnings +from OpenSSL import SSL from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer -from scrapy.core.downloader.tls import openssl_methods -from scrapy.utils.misc import create_instance, load_object -from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS +from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions +from scrapy.utils.misc import create_instance, load_object @implementer(IPolicyForHTTPS) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index d6e5cd1c1..650af9778 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -6,15 +6,15 @@ from urllib.parse import urldefrag from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.web.client import URI, BrowserLikePolicyForHTTPS +from twisted.web.client import BrowserLikePolicyForHTTPS, URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.python import to_bytes class H2DownloadHandler: @@ -88,29 +88,25 @@ class ScrapyH2Agent: if proxy: _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) scheme = _parse(request.url)[0] - proxy_host = str(proxy_host, 'utf-8') - omit_connect_timeout = b'noconnect' in proxy_params - if omit_connect_timeout: - warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Crawlera, it doesn't require this mode anymore, " - "so you should update scrapy-crawlera to 1.3.0+ " - "and remove '?noconnect' from the Crawlera URL.", - ScrapyDeprecationWarning) + proxy_host = proxy_host.decode() + omit_connect_tunnel = b'noconnect' in proxy_params + if omit_connect_tunnel: + warnings.warn("Using HTTPS proxies in the noconnect mode is not supported by the " + "downloader handler. If you use Crawlera, it doesn't require this " + "mode anymore, so you should update scrapy-crawlera to 1.3.0+ " + "and remove '?noconnect' from the Crawlera URL.") - if scheme == b'https' and not omit_connect_timeout: - proxy_auth = request.headers.get(b'Proxy-Authorization', None) - proxy_conf = (proxy_host, proxy_port, proxy_auth) - - # TODO: Return TunnelingAgent instance - else: - return self._ProxyAgent( - reactor=reactor, - context_factory=self._context_factory, - proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')), - connect_timeout=timeout, - bind_address=bind_address, - pool=self._pool - ) + if scheme == b'https' and not omit_connect_tunnel: + # ToDo + raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported') + return self._ProxyAgent( + reactor=reactor, + context_factory=self._context_factory, + proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')), + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool + ) return self._Agent( reactor=reactor, diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 40ea07b63..1e136fbd5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -192,10 +192,7 @@ class Stream: # a path component; these MUST include a ":path" pseudo-header field # with a value of '*' (refer RFC 7540 - Section 8.1.2.3) if not path: - if self._request.method == 'OPTIONS': - path = path or '*' - else: - path = path or '/' + path = '*' if self._request.method == 'OPTIONS' else '/' # Make sure pseudo-headers comes before all the other headers headers = [ diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 486614121..1cedf6b10 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -315,8 +315,8 @@ class HttpTestCase(unittest.TestCase): host = self.host + ':' + str(self.portno) def _test(response): - self.assertEqual(response.body, bytes(host, 'utf-8')) - self.assertEqual(request.headers.get('Host'), bytes(host, 'utf-8')) + self.assertEqual(response.body, to_bytes(host)) + self.assertEqual(request.headers.get('Host'), to_bytes(host)) request = Request(self.getURL('host'), headers={'Host': host}) return self.download_request(request, Spider('foo')).addCallback(_test) @@ -764,6 +764,7 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): download_handler_cls = HTTPDownloadHandler + expected_http_proxy_request_body = b'http://example.com' def setUp(self): site = server.Site(UriResource(), timeout=None) @@ -786,10 +787,7 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertTrue( - response.body == b'http://example.com' # HTTP/1.x - or response.body == b'/' # HTTP/2 - ) + self.assertEqual(response.body, self.expected_http_proxy_request_body) http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) @@ -799,13 +797,10 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertTrue( - response.body == b'http://example.com' # HTTP/1.x - or response.body == b'/' # HTTP/2 - ) + self.assertEqual(response.body, b'https://example.com') http_proxy = '%s?noconnect' % self.getURL('') - request = Request('http://example.com', meta={'proxy': http_proxy}) + request = Request('https://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex(ScrapyDeprecationWarning, r'Using HTTPS proxies in the noconnect mode is deprecated'): return self.download_request(request, Spider('foo')).addCallback(_test) @@ -851,6 +846,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase): host = u'127.0.0.1' download_handler_cls = H2DownloadHandler + expected_http_proxy_request_body = b'/' def setUp(self): site = server.Site(UriResource(), timeout=None) @@ -866,6 +862,21 @@ class Https2ProxyTestCase(Http11ProxyTestCase): def getURL(self, path): return f"{self.scheme}://{self.host}:{self.portno}/{path}" + def test_download_with_proxy_https_noconnect(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'/') + + http_proxy = '%s?noconnect' % self.getURL('') + request = Request('https://example.com', meta={'proxy': http_proxy}) + with self.assertWarnsRegex( + Warning, + r'Using HTTPS proxies in the noconnect mode is not supported by the ' + r'downloader handler.' + ): + return self.download_request(request, Spider('foo')).addCallback(_test) + class HttpDownloadHandlerMock: From 90f85a2b9b0cfac4a7a56a1926e9694ef7e2d299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Aug 2020 10:20:30 +0200 Subject: [PATCH 046/114] Enable Travis CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b403ac54c..a3bd2e199 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ dist: xenial branches: only: - master + - http2 # Remove once merged into master - /^\d\.\d+$/ - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: From af73f141b23aabbef208a12fe95ae63c27105fda Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 16 Aug 2020 11:26:10 +0530 Subject: [PATCH 047/114] refactor: move all http2 tests in separate files --- tests/test_download_handlers_http2.py | 158 ++++++++++++++++++++++++++ tests/test_downloader_handlers.py | 141 +---------------------- 2 files changed, 160 insertions(+), 139 deletions(-) create mode 100644 tests/test_download_handlers_http2.py diff --git a/tests/test_download_handlers_http2.py b/tests/test_download_handlers_http2.py new file mode 100644 index 000000000..583dc1d17 --- /dev/null +++ b/tests/test_download_handlers_http2.py @@ -0,0 +1,158 @@ +from unittest import mock + +from twisted.internet import defer, error, reactor +from twisted.trial import unittest +from twisted.web import server + +from scrapy.core.downloader.handlers.http2 import H2DownloadHandler +from scrapy.http import Request +from scrapy.spiders import Spider +from scrapy.utils.misc import create_instance +from scrapy.utils.test import get_crawler +from tests.mockserver import ssl_context_factory +from tests.test_downloader_handlers import ( + Https11TestCase, Https11CustomCiphers, + Http11MockServerTestCase, Http11ProxyTestCase, + UriResource +) + + +class Https2TestCase(Https11TestCase): + scheme = 'https' + download_handler_cls = H2DownloadHandler + HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + + @defer.inlineCallbacks + def test_download_with_maxsize_very_large_file(self): + with mock.patch('scrapy.core.http2.stream.logger') as logger: + request = Request(self.getURL('largechunkedfile')) + + def check(logger): + logger.error.assert_called_once_with(mock.ANY) + + d = self.download_request(request, Spider('foo', download_maxsize=1500)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + # As the error message is logged in the dataReceived callback, we + # have to give a bit of time to the reactor to process the queue + # after closing the connection. + d = defer.Deferred() + d.addCallback(check) + reactor.callLater(.1, d.callback, logger) + yield d + + def test_download_broken_content_cause_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_cause_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss_via_setting(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + +class Https2WrongHostnameTestCase(Https2TestCase): + tls_log_message = ( + 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' + 'subject "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' + ) + + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = 'keys/example-com.key.pem' + certfile = 'keys/example-com.cert.pem' + + +class Https2InvalidDNSId(Https2TestCase): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super(Https2InvalidDNSId, self).setUp() + self.host = '127.0.0.1' + + +class Https2InvalidDNSPattern(Https2TestCase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = 'keys/localhost.ip.key' + certfile = 'keys/localhost.ip.crt' + + def setUp(self): + try: + from service_identity.exceptions import CertificateError # noqa: F401 + except ImportError: + raise unittest.SkipTest("cryptography lib is too old") + self.tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' + 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' + ) + super(Https2InvalidDNSPattern, self).setUp() + + +class Https2CustomCiphers(Https11CustomCiphers): + scheme = 'https' + download_handler_cls = H2DownloadHandler + + +class Http2MockServerTestCase(Http11MockServerTestCase): + """HTTP 2.0 test case with MockServer""" + + +class Https2ProxyTestCase(Http11ProxyTestCase): + # only used for HTTPS tests + keyfile = 'keys/localhost.key' + certfile = 'keys/localhost.crt' + + scheme = 'https' + host = u'127.0.0.1' + + download_handler_cls = H2DownloadHandler + expected_http_proxy_request_body = b'/' + + def setUp(self): + site = server.Site(UriResource(), timeout=None) + self.port = reactor.listenSSL( + 0, site, + ssl_context_factory(self.keyfile, self.certfile), + interface=self.host + ) + self.portno = self.port.getHost().port + self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) + self.download_request = self.download_handler.download_request + + def getURL(self, path): + return f"{self.scheme}://{self.host}:{self.portno}/{path}" + + def test_download_with_proxy_https_noconnect(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'/') + + http_proxy = '%s?noconnect' % self.getURL('') + request = Request('https://example.com', meta={'proxy': http_proxy}) + with self.assertWarnsRegex( + Warning, + r'Using HTTPS proxies in the noconnect mode is not supported by the ' + r'downloader handler.' + ): + return self.download_request(request, Spider('foo')).addCallback(_test) + + @defer.inlineCallbacks + def test_download_with_proxy_https_timeout(self): + with self.assertRaises(NotImplementedError): + yield super(Https2ProxyTestCase, self).test_download_with_proxy_https_timeout() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 1cedf6b10..8b2b2c32c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -23,7 +23,6 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler -from scrapy.core.downloader.handlers.http2 import H2DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Headers, Request @@ -315,8 +314,8 @@ class HttpTestCase(unittest.TestCase): host = self.host + ':' + str(self.portno) def _test(response): - self.assertEqual(response.body, to_bytes(host)) - self.assertEqual(request.headers.get('Host'), to_bytes(host)) + self.assertEqual(response.body, host.encode()) + self.assertEqual(request.headers.get('Host'), host.encode()) request = Request(self.getURL('host'), headers={'Host': host}) return self.download_request(request, Spider('foo')).addCallback(_test) @@ -522,49 +521,6 @@ class Https11TestCase(Http11TestCase): yield download_handler.close() -class Https2TestCase(Https11TestCase): - scheme = 'https' - download_handler_cls = H2DownloadHandler - HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - - @defer.inlineCallbacks - def test_download_with_maxsize_very_large_file(self): - with mock.patch('scrapy.core.http2.stream.logger') as logger: - request = Request(self.getURL('largechunkedfile')) - - def check(logger): - logger.error.assert_called_once_with(mock.ANY) - - d = self.download_request(request, Spider('foo', download_maxsize=1500)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - # As the error message is logged in the dataReceived callback, we - # have to give a bit of time to the reactor to process the queue - # after closing the connection. - d = defer.Deferred() - d.addCallback(check) - reactor.callLater(.1, d.callback, logger) - yield d - - def test_download_broken_content_cause_data_loss(self, url='broken'): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_chunked_content_cause_data_loss(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_content_allow_data_loss(self, url='broken'): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_chunked_content_allow_data_loss(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_chunked_content_allow_data_loss_via_setting(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - class Https11WrongHostnameTestCase(Http11TestCase): scheme = 'https' @@ -577,23 +533,6 @@ class Https11WrongHostnameTestCase(Http11TestCase): certfile = 'keys/example-com.cert.pem' -class Https2WrongHostnameTestCase(Https2TestCase): - tls_log_message = ( - 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' - 'subject "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' - ) - - # above tests use a server certificate for "localhost", - # client connection to "localhost" too. - # here we test that even if the server certificate is for another domain, - # "www.example.com" in this case, - # the tests still pass - keyfile = 'keys/example-com.key.pem' - certfile = 'keys/example-com.cert.pem' - - class Https11InvalidDNSId(Https11TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" @@ -602,14 +541,6 @@ class Https11InvalidDNSId(Https11TestCase): self.host = '127.0.0.1' -class Https2InvalidDNSId(Https2TestCase): - """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" - - def setUp(self): - super(Https2InvalidDNSId, self).setUp() - self.host = '127.0.0.1' - - class Https11InvalidDNSPattern(Https11TestCase): """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" @@ -628,24 +559,6 @@ class Https11InvalidDNSPattern(Https11TestCase): super(Https11InvalidDNSPattern, self).setUp() -class Https2InvalidDNSPattern(Https2TestCase): - """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - - keyfile = 'keys/localhost.ip.key' - certfile = 'keys/localhost.ip.crt' - - def setUp(self): - try: - from service_identity.exceptions import CertificateError # noqa: F401 - except ImportError: - raise unittest.SkipTest("cryptography lib is too old") - self.tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' - 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' - ) - super(Https2InvalidDNSPattern, self).setUp() - - class Https11CustomCiphers(unittest.TestCase): scheme = 'https' download_handler_cls = HTTP11DownloadHandler @@ -686,11 +599,6 @@ class Https11CustomCiphers(unittest.TestCase): return d -class Https2CustomCiphers(Https11CustomCiphers): - scheme = 'https' - download_handler_cls = H2DownloadHandler - - class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" @@ -742,10 +650,6 @@ class Http11MockServerTestCase(unittest.TestCase): self.assertTrue(reason, 'finished') -class Http2MockServerTestCase(Http11MockServerTestCase): - """HTTP 2.0 test case with MockServer""" - - class UriResource(resource.Resource): """Return the full uri that was requested""" @@ -837,47 +741,6 @@ class Http11ProxyTestCase(HttpProxyTestCase): self.assertIn(domain, timeout.osError) -class Https2ProxyTestCase(Http11ProxyTestCase): - # only used for HTTPS tests - keyfile = 'keys/localhost.key' - certfile = 'keys/localhost.crt' - - scheme = 'https' - host = u'127.0.0.1' - - download_handler_cls = H2DownloadHandler - expected_http_proxy_request_body = b'/' - - def setUp(self): - site = server.Site(UriResource(), timeout=None) - self.port = reactor.listenSSL( - 0, site, - ssl_context_factory(self.keyfile, self.certfile), - interface=self.host - ) - self.portno = self.port.getHost().port - self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) - self.download_request = self.download_handler.download_request - - def getURL(self, path): - return f"{self.scheme}://{self.host}:{self.portno}/{path}" - - def test_download_with_proxy_https_noconnect(self): - def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'/') - - http_proxy = '%s?noconnect' % self.getURL('') - request = Request('https://example.com', meta={'proxy': http_proxy}) - with self.assertWarnsRegex( - Warning, - r'Using HTTPS proxies in the noconnect mode is not supported by the ' - r'downloader handler.' - ): - return self.download_request(request, Spider('foo')).addCallback(_test) - - class HttpDownloadHandlerMock: def __init__(self, *args, **kwargs): From f9f008e935c5c892c2839d354ded3ee213057d9e Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 16 Aug 2020 17:04:40 +0530 Subject: [PATCH 048/114] test: add typing-extensions --- .travis.yml | 2 +- setup.py | 1 + tox.ini | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e273e358d..0b55cda19 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ dist: xenial branches: only: - master - - http2 # Remove once merged into master + - http2 # ToDo: Remove once merged into master - /^\d\.\d+$/ - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: diff --git a/setup.py b/setup.py index 7f4ff0095..c8733ae96 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'h2>=3.2.0', + 'typing-extensions>=3.7', ] extras_require = {} diff --git a/tox.ini b/tox.ini index e8fdbd85d..3bf2224ec 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,7 @@ deps = -ctests/constraints.txt -rtests/requirements-py3.txt # Extras + Twisted[http2]>=17.9.0 boto3>=1.13.0 botocore>=1.3.23 Pillow>=3.4.2 @@ -64,6 +65,7 @@ deps = -ctests/constraints.txt cryptography==2.0 cssselect==0.9.1 + h2==3.2.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 @@ -72,12 +74,12 @@ deps = queuelib==1.4.2 service_identity==16.0.0 Twisted[http2]==17.9.0 + typing-extensions==3.7 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt # Extras botocore==1.3.23 - h2==3.2.0 google-cloud-storage==1.29.0 Pillow==3.4.2 From 38d361792c02ae2b25323258d070c04d8906495a Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 16 Aug 2020 17:55:16 +0530 Subject: [PATCH 049/114] fix: typing & pylint errors - Ignore typing check for http2 test files --- scrapy/core/downloader/handlers/http2.py | 4 ++-- scrapy/core/http2/protocol.py | 2 +- setup.cfg | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 650af9778..f2ed40f9b 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -71,8 +71,8 @@ class ScrapyH2Agent: def __init__( self, context_factory, - connect_timeout=10, - bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None, + pool: H2ConnectionPool, + connect_timeout=10, bind_address: Optional[bytes] = None, crawler=None ) -> None: self._context_factory = context_factory diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 1ce8b6548..fee391af6 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -323,7 +323,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) elif isinstance(event, UnknownFrameReceived): - logger.debug(f'UnknownFrameReceived: frame={event.frame}') + logger.debug('UnknownFrameReceived: frame={}'.format(event.frame)) # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated) -> None: diff --git a/setup.cfg b/setup.cfg index f8e7c0c91..8b70a0e60 100644 --- a/setup.cfg +++ b/setup.cfg @@ -100,6 +100,9 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True +[mypy-tests.test_download_handlers_http2] +ignore_errors = True + [mypy-tests.test_engine] ignore_errors = True @@ -109,6 +112,9 @@ ignore_errors = True [mypy-tests.test_http_request] ignore_errors = True +[mypy-tests.test_http2_client_protocol] +ignore_errors = True + [mypy-tests.test_linkextractors] ignore_errors = True From 75fe3d13657e11bdabb9b26b452c6cdacecffec7 Mon Sep 17 00:00:00 2001 From: adityaa30 Date: Mon, 17 Aug 2020 03:47:17 +0530 Subject: [PATCH 050/114] fix: increase timeout to 0.5 seconds - In Windows specifically the reactor was left unclean by the HostnameEndpoint due to the tearDown method of test_downloader_handlers.py::HttpTestCase due to which the following 2 tests were failing: 1. test_timeout_download_from_spider_server_hangs 2. test_timeout_download_from_spider_nodata_rcvd - Increasing the timeout fixed the test (in local) --- setup.cfg | 2 +- tests/test_downloader_handlers.py | 4 ++-- ...ad_handlers_http2.py => test_downloader_handlers_http2.py} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename tests/{test_download_handlers_http2.py => test_downloader_handlers_http2.py} (100%) diff --git a/setup.cfg b/setup.cfg index 8b70a0e60..5b267c295 100644 --- a/setup.cfg +++ b/setup.cfg @@ -100,7 +100,7 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True -[mypy-tests.test_download_handlers_http2] +[mypy-tests.test_downloader_handlers_http2] ignore_errors = True [mypy-tests.test_engine] diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index ecac47d90..2b3fa2aca 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -287,7 +287,7 @@ class HttpTestCase(unittest.TestCase): def test_timeout_download_from_spider_nodata_rcvd(self): # client connects but no data is received spider = Spider('foo') - meta = {'download_timeout': 0.2} + meta = {'download_timeout': 0.5} request = Request(self.getURL('wait'), meta=meta) d = self.download_request(request, spider) yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) @@ -296,7 +296,7 @@ class HttpTestCase(unittest.TestCase): def test_timeout_download_from_spider_server_hangs(self): # client connects, server send headers and some body bytes but hangs spider = Spider('foo') - meta = {'download_timeout': 0.2} + meta = {'download_timeout': 0.5} request = Request(self.getURL('hang-after-headers'), meta=meta) d = self.download_request(request, spider) yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) diff --git a/tests/test_download_handlers_http2.py b/tests/test_downloader_handlers_http2.py similarity index 100% rename from tests/test_download_handlers_http2.py rename to tests/test_downloader_handlers_http2.py From a87ab71d1061585e41864d7283557bbe9823a91b Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 18 Aug 2020 04:47:09 +0530 Subject: [PATCH 051/114] refactor(http2): metadata for Stream - Add Note about HTTP/2 Cleartext not supported in settings.rst --- docs/topics/settings.rst | 7 +++ scrapy/core/http2/protocol.py | 2 +- scrapy/core/http2/stream.py | 73 +++++++++++++++---------------- scrapy/core/http2/types.py | 23 ++++++++++ setup.py | 2 +- tests/test_downloader_handlers.py | 2 +- 6 files changed, 68 insertions(+), 41 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d4d4f9332..0dad30b29 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -627,6 +627,13 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', } +.. note:: + + Scrapy currently does not support HTTP/2 Cleartext (h2c) since none + of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_). + +.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index fee391af6..7a3156541 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -289,7 +289,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._conn_lost_deferred.callback(self._conn_lost_errors) for stream in self.streams.values(): - if stream.request_sent: + if stream.metadata['request_sent']: close_reason = StreamCloseReason.CONNECTION_LOST else: close_reason = StreamCloseReason.INACTIVE diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 1e136fbd5..bddf50a56 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -5,14 +5,14 @@ from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse from h2.errors import ErrorCodes -from h2.exceptions import H2Error, StreamClosedError +from h2.exceptions import H2Error, StreamClosedError, ProtocolError from hpack import HeaderTuple from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from scrapy.core.http2.types import H2ResponseDict +from scrapy.core.http2.types import H2ResponseDict, H2StreamMetadataDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes @@ -100,24 +100,14 @@ class Stream: self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) - self.request_start_time = None - - self.content_length = 0 if self._request.body is None else len(self._request.body) - - # Flag to keep track whether this stream has initiated the request - self.request_sent = False - - # Flag to track whether we have logged about exceeding download warnsize - self._reached_warnsize = False - - # Each time we send a data frame, we will decrease value by the amount send. - self.remaining_content_length = self.content_length - - # Flag to keep track whether we have closed this stream - self.stream_closed_local = False - - # Flag to keep track whether the server has closed the stream - self.stream_closed_server = False + self.metadata: H2StreamMetadataDict = { + 'request_content_length': 0 if self._request.body is None else len(self._request.body), + 'request_sent': False, + 'reached_warnsize': False, + 'remaining_content_length': 0 if self._request.body is None else len(self._request.body), + 'stream_closed_local': False, + 'stream_closed_server': False, + } # Private variable used to build the response # this response is then converted to appropriate Response class @@ -132,7 +122,7 @@ class Stream: # Close this stream as gracefully as possible # If the associated request is initiated we reset this stream # else we directly call close() method - if self.request_sent: + if self.metadata['request_sent']: self.reset_stream(StreamCloseReason.CANCELLED) else: self.close(StreamCloseReason.CANCELLED) @@ -160,7 +150,7 @@ class Stream: self._response['flow_controlled_size'] > self._download_warnsize or content_length_header > self._download_warnsize ) - and not self._reached_warnsize + and not self.metadata['reached_warnsize'] ) def get_response(self) -> Deferred: @@ -220,7 +210,7 @@ class Stream: if self.check_request_url(): headers = self._get_request_headers() self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False) - self.request_sent = True + self.metadata['request_sent'] = True self.send_data() else: # Close this stream calling the response errback @@ -238,7 +228,7 @@ class Stream: and has initiated request already by sending HEADER frame. If not then stream will raise ProtocolError (raise by h2 state machine). """ - if self.stream_closed_local: + if self.metadata['stream_closed_local']: raise StreamClosedError(self.stream_id) # Firstly, check what the flow control window is for current stream. @@ -249,24 +239,24 @@ class Stream: # We will send no more than the window size or the remaining file size # of data in this call, whichever is smaller. - bytes_to_send_size = min(window_size, self.remaining_content_length) + bytes_to_send_size = min(window_size, self.metadata['remaining_content_length']) # We now need to send a number of data frames. while bytes_to_send_size > 0: chunk_size = min(bytes_to_send_size, max_frame_size) - data_chunk_start_id = self.content_length - self.remaining_content_length + data_chunk_start_id = self.metadata['request_content_length'] - self.metadata['remaining_content_length'] data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size] self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False) bytes_to_send_size = bytes_to_send_size - chunk_size - self.remaining_content_length = self.remaining_content_length - chunk_size + self.metadata['remaining_content_length'] = self.metadata['remaining_content_length'] - chunk_size - self.remaining_content_length = max(0, self.remaining_content_length) + self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length']) # End the stream if no more data needs to be send - if self.remaining_content_length == 0: + if self.metadata['remaining_content_length'] == 0: self._protocol.conn.end_stream(self.stream_id) # Q. What about the rest of the data? @@ -277,7 +267,11 @@ class Stream: Send data that earlier could not be sent as we were blocked behind the flow control. """ - if self.remaining_content_length and not self.stream_closed_server and self.request_sent: + if ( + self.metadata['remaining_content_length'] + and not self.metadata['stream_closed_server'] + and self.metadata['request_sent'] + ): self.send_data() def receive_data(self, data: bytes, flow_controlled_length: int) -> None: @@ -290,7 +284,7 @@ class Stream: return if self._log_warnsize: - self._reached_warnsize = True + self.metadata['reached_warnsize'] = True warning_msg = ( f'Received more ({self._response["flow_controlled_size"]}) bytes than download ' f'warn size ({self._download_warnsize}) in request {self._request}' @@ -314,7 +308,7 @@ class Stream: return if self._log_warnsize: - self._reached_warnsize = True + self.metadata['reached_warnsize'] = True warning_msg = ( f'Expected response size ({expected_size}) larger than ' f'download warn size ({self._download_warnsize}) in request {self._request}' @@ -323,18 +317,18 @@ class Stream: def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None: """Close this stream by sending a RST_FRAME to the remote peer""" - if self.stream_closed_local: + if self.metadata['stream_closed_local']: raise StreamClosedError(self.stream_id) # Clear buffer earlier to avoid keeping data in memory for a long time self._response['body'].truncate(0) - self.stream_closed_local = True + self.metadata['stream_closed_local'] = True self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) self.close(reason) def _is_data_lost(self) -> bool: - assert self.stream_closed_server + assert self.metadata['stream_closed_server'] expected_size = self._response['flow_controlled_size'] received_body_size = int(self._response['headers'][b'Content-Length']) @@ -349,7 +343,7 @@ class Stream: ) -> None: """Based on the reason sent we will handle each case. """ - if self.stream_closed_server: + if self.metadata['stream_closed_server']: raise StreamClosedError(self.stream_id) if not isinstance(reason, StreamCloseReason): @@ -362,7 +356,7 @@ class Stream: if not from_protocol: self._protocol.pop_stream(self.stream_id) - self.stream_closed_server = True + self.metadata['stream_closed_server'] = True # We do not check for Content-Length or Transfer-Encoding in response headers # and add `partial` flag as in HTTP/1.1 as 'A request or response that includes @@ -402,7 +396,10 @@ class Stream: elif reason is StreamCloseReason.RESET: self._deferred_response.errback(ResponseFailed([ - Failure(f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM') + Failure( + f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM', + ProtocolError + ) ])) elif reason is StreamCloseReason.CONNECTION_LOST: diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py index d2aa1a9d8..ff8d94066 100644 --- a/scrapy/core/http2/types.py +++ b/scrapy/core/http2/types.py @@ -31,6 +31,29 @@ class H2ConnectionMetadataDict(TypedDict): default_download_warnsize: int +class H2StreamMetadataDict(TypedDict): + """Metadata of an HTTP/2 connection stream + initialized when stream is instantiated + """ + + request_content_length: int + + # Flag to keep track whether the stream has initiated the request + request_sent: bool + + # Flag to track whether we have logged about exceeding download warnsize + reached_warnsize: bool + + # Each time we send a data frame, we will decrease value by the amount send. + remaining_content_length: int + + # Flag to keep track whether we have closed this stream + stream_closed_local: bool + + # Flag to keep track whether the server has closed the stream + stream_closed_server: bool + + class H2ResponseDict(TypedDict): # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. diff --git a/setup.py b/setup.py index c8733ae96..66f369a71 100644 --- a/setup.py +++ b/setup.py @@ -97,4 +97,4 @@ setup( python_requires='>=3.5.2', install_requires=install_requires, extras_require=extras_require, -) \ No newline at end of file +) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 2b3fa2aca..e3777ee1d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -232,7 +232,7 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"echo", Echo()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) - self.host = u'localhost' + self.host = 'localhost' if self.scheme == 'https': # Using WrappingFactory do not enable HTTP/2 failing all the # tests with H2DownloadHandler From a206ac5f6f4e5eb057ecc535556a15087b250d40 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 18 Aug 2020 07:36:00 +0530 Subject: [PATCH 052/114] tests: disable python 3.5 for travis and azure --- .travis.yml | 25 +++++++++++++------------ azure-pipelines.yml | 7 ++++--- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0b55cda19..6628e8e43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,18 +18,19 @@ matrix: python: 3.7 # Keep in sync with .readthedocs.yml - env: TOXENV=typing python: 3.8 - - - env: TOXENV=pinned - python: 3.5.2 - - env: TOXENV=asyncio-pinned - python: 3.5.2 # We use additional code to support 3.5.3 and earlier - - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0 - - - env: TOXENV=py - python: 3.5 - - env: TOXENV=asyncio - python: 3.5 # We use specific code to support >= 3.5.4, < 3.6 - - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0 + +# ToDo: Remove once merged into master +# - env: TOXENV=pinned +# python: 3.5.2 +# - env: TOXENV=asyncio-pinned +# python: 3.5.2 # We use additional code to support 3.5.3 and earlier +# - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0 +# +# - env: TOXENV=py +# python: 3.5 +# - env: TOXENV=asyncio +# python: 3.5 # We use specific code to support >= 3.5.4, < 3.6 +# - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0 - env: TOXENV=py python: 3.6 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 710e42090..c77c128b3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -4,9 +4,10 @@ pool: vmImage: 'windows-latest' strategy: matrix: - Python35: - python.version: '3.5' - TOXENV: windows-pinned +# ToDo: Remove once merged into master +# Python35: +# python.version: '3.5' +# TOXENV: windows-pinned Python36: python.version: '3.6' Python37: From e3233b79deee6ad283ed4316135e30ab774c1078 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 19 Aug 2020 05:10:19 +0530 Subject: [PATCH 053/114] refactor(h2-stream): alphabetical order of imports --- scrapy/core/downloader/handlers/http2.py | 2 +- scrapy/core/http2/stream.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index f2ed40f9b..ddd813cec 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -29,7 +29,7 @@ class H2DownloadHandler: def from_crawler(cls, crawler): return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index bddf50a56..33302421e 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -5,7 +5,7 @@ from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse from h2.errors import ErrorCodes -from h2.exceptions import H2Error, StreamClosedError, ProtocolError +from h2.exceptions import H2Error, ProtocolError, StreamClosedError from hpack import HeaderTuple from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed From 30eb005639b77ea94e8859e1d041dfe53048cf77 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 19 Aug 2020 06:25:04 +0530 Subject: [PATCH 054/114] fix: InvalidNegotiatedProtocol __str__ method --- scrapy/core/http2/agent.py | 2 +- scrapy/core/http2/protocol.py | 2 +- scrapy/core/http2/stream.py | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index f4ac29bc6..f829cc5f8 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -68,7 +68,7 @@ class H2ConnectionPool: # Now as we have established a proper HTTP/2 connection # we fire all the deferred's with the connection instance - pending_requests = self._pending_requests.pop(key) + pending_requests = self._pending_requests.pop(key, None) while pending_requests: d = pending_requests.popleft() d.callback(conn) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 7a3156541..c6f00423a 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -39,7 +39,7 @@ class InvalidNegotiatedProtocol(H2Error): self.negotiated_protocol = negotiated_protocol def __str__(self) -> str: - return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}' + return f'InvalidNegotiatedProtocol: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}' class RemoteTerminatedConnection(H2Error): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 33302421e..df7470e11 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -31,6 +31,9 @@ class InactiveStreamClosed(ConnectionClosed): def __init__(self, request: Request): self.request = request + def __str__(self) -> str: + return f'InactiveStreamClosed: Connection was closed without sending the request {self.request!r}' + class InvalidHostname(H2Error): From 2f00666d749b4a736536bafc786e63be06113676 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 19 Aug 2020 07:31:52 +0530 Subject: [PATCH 055/114] refactor: move agents & context-factory --- scrapy/core/downloader/contextfactory.py | 22 ++++++++++++- scrapy/core/downloader/handlers/http2.py | 31 ++--------------- scrapy/core/http2/agent.py | 42 ++++++++++++++---------- 3 files changed, 49 insertions(+), 46 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 5768d8f8e..073ef16bf 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,10 +1,12 @@ import warnings from OpenSSL import SSL +from twisted.internet._sslverify import _setAcceptableProtocols from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer +from zope.interface.verify import verifyObject from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions from scrapy.utils.misc import create_instance, load_object @@ -84,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): The default OpenSSL method is ``TLS_METHOD`` (also called ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ - def creatorForNetloc(self, hostname, port): + def creatorForNetloc(self, hostname, port): # trustRoot set to platformTrust() will use the platform's root CAs. # # This means that a website like https://www.cacert.org will be rejected @@ -97,6 +99,24 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): ) +@implementer(IPolicyForHTTPS) +class AcceptableProtocolsContextFactory: + """Context factory to used to override the acceptable protocols + to set up the [OpenSSL.SSL.Context] for doing NPN and/or ALPN + negotiation. + """ + + def __init__(self, context_factory, acceptable_protocols): + verifyObject(IPolicyForHTTPS, context_factory) + self._wrapped_context_factory = context_factory + self._acceptable_protocols = acceptable_protocols + + def creatorForNetloc(self, hostname, port): + options = self._wrapped_context_factory.creatorForNetloc(hostname, port) + _setAcceptableProtocols(options._ctx, self._acceptable_protocols) + return options + + def load_context_factory_from_settings(settings, crawler): ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index ddd813cec..4be888bda 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,16 +1,15 @@ import warnings from time import time -from typing import Optional, Tuple +from typing import Optional from urllib.parse import urldefrag -from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.web.client import BrowserLikePolicyForHTTPS, URI +from twisted.web.client import URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse -from scrapy.core.http2.agent import H2Agent, H2ConnectionPool +from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider @@ -41,30 +40,6 @@ class H2DownloadHandler: self._pool.close_connections() -class ScrapyProxyH2Agent(H2Agent): - def __init__( - self, reactor: ReactorBase, - proxy_uri: URI, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None - ) -> None: - super(ScrapyProxyH2Agent, self).__init__( - reactor=reactor, - pool=pool, - context_factory=context_factory, - connect_timeout=connect_timeout, - bind_address=bind_address - ) - self._proxy_uri = proxy_uri - - def get_endpoint(self, uri: URI): - return self.endpoint_factory.endpointForURI(self._proxy_uri) - - def get_key(self, uri: URI) -> Tuple: - """We use the proxy uri instead of uri obtained from request url""" - return "http-proxy", self._proxy_uri.host, self._proxy_uri.port - - class ScrapyH2Agent: _Agent = H2Agent _ProxyAgent = ScrapyProxyH2Agent diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index f829cc5f8..d950c6cfb 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -2,17 +2,14 @@ from collections import deque from typing import Deque, Dict, List, Tuple, Optional from twisted.internet import defer -from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.endpoints import HostnameEndpoint from twisted.python.failure import Failure from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory from twisted.web.error import SchemeNotSupported -from twisted.web.iweb import IPolicyForHTTPS -from zope.interface import implementer -from zope.interface.verify import verifyObject +from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory from scrapy.http.request import Request from scrapy.settings import Settings @@ -96,19 +93,6 @@ class H2ConnectionPool: conn.transport.abortConnection() -@implementer(IPolicyForHTTPS) -class AcceptableProtocolsContextFactory: - def __init__(self, context_factory, acceptable_protocols: List[bytes]) -> None: - verifyObject(IPolicyForHTTPS, context_factory) - self._wrapped_context_factory = context_factory - self._acceptable_protocols = acceptable_protocols - - def creatorForNetloc(self, hostname, port) -> ClientTLSOptions: - options = self._wrapped_context_factory.creatorForNetloc(hostname, port) - _setAcceptableProtocols(options._ctx, self._acceptable_protocols) - return options - - class H2Agent: def __init__( self, reactor: ReactorBase, pool: H2ConnectionPool, @@ -144,3 +128,27 @@ class H2Agent: d = self._pool.get_connection(key, uri, endpoint) d.addCallback(lambda conn: conn.request(request, spider)) return d + + +class ScrapyProxyH2Agent(H2Agent): + def __init__( + self, reactor: ReactorBase, + proxy_uri: URI, pool: H2ConnectionPool, + context_factory=BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + ) -> None: + super(ScrapyProxyH2Agent, self).__init__( + reactor=reactor, + pool=pool, + context_factory=context_factory, + connect_timeout=connect_timeout, + bind_address=bind_address + ) + self._proxy_uri = proxy_uri + + def get_endpoint(self, uri: URI): + return self.endpoint_factory.endpointForURI(self._proxy_uri) + + def get_key(self, uri: URI) -> Tuple: + """We use the proxy uri instead of uri obtained from request url""" + return "http-proxy", self._proxy_uri.host, self._proxy_uri.port From 1432161477673152f4d2f95cd32829a81ffe3f70 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 24 Aug 2020 15:40:01 +0530 Subject: [PATCH 056/114] fix: bump min typing-extensions version to 3.7.4 - typing-extensions>=3.7.4 only supports TypedDict --- setup.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 607f1dadf..39482d383 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'h2>=3.2.0', - 'typing-extensions>=3.7', + 'typing-extensions>=3.7.4', ] extras_require = {} diff --git a/tox.ini b/tox.ini index 3f6fd1224..0a88ed8af 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,7 @@ deps = queuelib==1.4.2 service_identity==16.0.0 Twisted[http2]==17.9.0 - typing-extensions==3.7 + typing-extensions==3.7.4 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt From 450ba6b51f7cc3cd89a92e8ff4d9e105865eb862 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 26 Aug 2020 17:20:59 +0530 Subject: [PATCH 057/114] fix(typo): stream -> streams, use isinstance --- scrapy/core/http2/protocol.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index c6f00423a..6647ae0b7 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -90,7 +90,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # all requests in a pool and send them as the connection is made self._pending_request_stream_pool: deque = deque() - # Counter to keep track of opened stream. This counter + # Counter to keep track of opened streams. This counter # is used to make sure that not more than MAX_CONCURRENT_STREAMS # streams are opened which leads to ProtocolError # We use simple FIFO policy to handle pending requests @@ -218,7 +218,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """We close the connection with InvalidNegotiatedProtocol exception when the connection was not made via h2 protocol""" negotiated_protocol = self.transport.negotiatedProtocol - if type(negotiated_protocol) is bytes: + if isinstance(negotiated_protocol, bytes): negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') if negotiated_protocol != 'h2': # Here we have not initiated the connection yet From 5e36f539e28631625862b84d8ca1c7c0134584b0 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 27 Aug 2020 15:12:22 +0530 Subject: [PATCH 058/114] chore: remove typing-extensions dependency --- scrapy/core/http2/protocol.py | 52 ++++++++++++++++----------- scrapy/core/http2/stream.py | 27 +++++++++++--- scrapy/core/http2/types.py | 67 ----------------------------------- setup.py | 1 - tox.ini | 1 - 5 files changed, 55 insertions(+), 93 deletions(-) delete mode 100644 scrapy/core/http2/types.py diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 6647ae0b7..0b872f6ba 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -3,7 +3,6 @@ import itertools import logging from collections import deque from ipaddress import IPv4Address, IPv6Address -from typing import Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -22,10 +21,10 @@ from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin from twisted.python.failure import Failure from twisted.web.client import URI +from typing import Dict, List, Optional, Union from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason -from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request from scrapy.settings import Settings from scrapy.spiders import Spider @@ -90,26 +89,39 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # all requests in a pool and send them as the connection is made self._pending_request_stream_pool: deque = deque() - # Counter to keep track of opened streams. This counter - # is used to make sure that not more than MAX_CONCURRENT_STREAMS - # streams are opened which leads to ProtocolError - # We use simple FIFO policy to handle pending requests - self._active_streams = 0 - - # Flag to keep track if settings were acknowledged by the remote - # This ensures that we have established a HTTP/2 connection - self._settings_acknowledged = False - # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure self._conn_lost_errors: List[BaseException] = [] - self.metadata: H2ConnectionMetadataDict = { + # Some meta data of this connection + # initialized when connection is successfully made + self.metadata: Dict = { + # Peer certificate instance 'certificate': None, + + # Address of the server we are connected to which + # is updated when HTTP/2 connection is made successfully 'ip_address': None, + + # URI of the peer HTTP/2 connection is made 'uri': uri, + + # Both ip_address and uri are used by the Stream before + # initiating the request to verify that the base address + + # Variables taken from Project Settings 'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'), 'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'), + + # Counter to keep track of opened streams. This counter + # is used to make sure that not more than MAX_CONCURRENT_STREAMS + # streams are opened which leads to ProtocolError + # We use simple FIFO policy to handle pending requests + 'active_streams': 0, + + # Flag to keep track if settings were acknowledged by the remote + # This ensures that we have established a HTTP/2 connection + 'settings_acknowledged': False, } @property @@ -118,7 +130,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection """ - return bool(self.transport.connected) and self._settings_acknowledged + return bool(self.transport.connected) and self.metadata['settings_acknowledged'] @property def allowed_max_concurrent_streams(self) -> int: @@ -139,10 +151,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """ while ( self._pending_request_stream_pool - and self._active_streams < self.allowed_max_concurrent_streams + and self.metadata['active_streams'] < self.allowed_max_concurrent_streams and self.h2_connected ): - self._active_streams += 1 + self.metadata['active_streams'] += 1 stream = self._pending_request_stream_pool.popleft() stream.initiate_request() self._write_to_transport() @@ -151,7 +163,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """Perform cleanup when a stream is closed """ stream = self.streams.pop(stream_id) - self._active_streams -= 1 + self.metadata['active_streams'] -= 1 self._send_pending_requests() return stream @@ -261,7 +273,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): if ( self.conn.open_outbound_streams > 0 or self.conn.open_inbound_streams > 0 - or self._active_streams > 0 + or self.metadata['active_streams'] > 0 ): error_code = ErrorCodes.PROTOCOL_ERROR else: @@ -295,7 +307,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): close_reason = StreamCloseReason.INACTIVE stream.close(close_reason, self._conn_lost_errors, from_protocol=True) - self._active_streams -= len(self.streams) + self.metadata['active_streams'] -= len(self.streams) self.streams.clear() self._pending_request_stream_pool.clear() self.conn.close_connection() @@ -338,7 +350,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.streams[event.stream_id].receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: - self._settings_acknowledged = True + self.metadata['settings_acknowledged'] = True # Send off all the pending requests as now we have # established a proper HTTP/2 connection diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index df7470e11..e01e76ae5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,7 +1,6 @@ import logging from enum import Enum from io import BytesIO -from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse from h2.errors import ErrorCodes @@ -11,8 +10,9 @@ from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed +from typing import Dict +from typing import List, Optional, Tuple, TYPE_CHECKING -from scrapy.core.http2.types import H2ResponseDict, H2StreamMetadataDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes @@ -103,21 +103,40 @@ class Stream: self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) - self.metadata: H2StreamMetadataDict = { + # Metadata of an HTTP/2 connection stream + # initialized when stream is instantiated + self.metadata: Dict = { 'request_content_length': 0 if self._request.body is None else len(self._request.body), + + # Flag to keep track whether the stream has initiated the request 'request_sent': False, + + # Flag to track whether we have logged about exceeding download warnsize 'reached_warnsize': False, + + # Each time we send a data frame, we will decrease value by the amount send. 'remaining_content_length': 0 if self._request.body is None else len(self._request.body), + + # Flag to keep track whether client (self) have closed this stream 'stream_closed_local': False, + + # Flag to keep track whether the server has closed the stream 'stream_closed_server': False, } # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: H2ResponseDict = { + self._response: Dict = { + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. 'body': BytesIO(), + + # The amount of data received that counts against the + # flow control window 'flow_controlled_size': 0, + + # Headers received after sending the request 'headers': Headers({}), } diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py deleted file mode 100644 index ff8d94066..000000000 --- a/scrapy/core/http2/types.py +++ /dev/null @@ -1,67 +0,0 @@ -from io import BytesIO -from ipaddress import IPv4Address, IPv6Address -from typing import Union, Optional - -from twisted.internet.ssl import Certificate -from twisted.web.client import URI -# for python < 3.8 -- typing.TypedDict is undefined -from typing_extensions import TypedDict - -from scrapy.http.headers import Headers - - -class H2ConnectionMetadataDict(TypedDict): - """Some meta data of this connection - initialized when connection is successfully made - """ - certificate: Optional[Certificate] - - # Address of the server we are connected to which - # is updated when HTTP/2 connection is made successfully - ip_address: Optional[Union[IPv4Address, IPv6Address]] - - # URI of the peer HTTP/2 connection is made - uri: URI - - # Both ip_address and uri are used by the Stream before - # initiating the request to verify that the base address - - # Variables taken from Project Settings - default_download_maxsize: int - default_download_warnsize: int - - -class H2StreamMetadataDict(TypedDict): - """Metadata of an HTTP/2 connection stream - initialized when stream is instantiated - """ - - request_content_length: int - - # Flag to keep track whether the stream has initiated the request - request_sent: bool - - # Flag to track whether we have logged about exceeding download warnsize - reached_warnsize: bool - - # Each time we send a data frame, we will decrease value by the amount send. - remaining_content_length: int - - # Flag to keep track whether we have closed this stream - stream_closed_local: bool - - # Flag to keep track whether the server has closed the stream - stream_closed_server: bool - - -class H2ResponseDict(TypedDict): - # Data received frame by frame from the server is appended - # and passed to the response Deferred when completely received. - body: BytesIO - - # The amount of data received that counts against the flow control - # window - flow_controlled_size: int - - # Headers received after sending the request - headers: Headers diff --git a/setup.py b/setup.py index 39482d383..34af7ec67 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,6 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'h2>=3.2.0', - 'typing-extensions>=3.7.4', ] extras_require = {} diff --git a/tox.ini b/tox.ini index 0a88ed8af..78090d6de 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,6 @@ deps = queuelib==1.4.2 service_identity==16.0.0 Twisted[http2]==17.9.0 - typing-extensions==3.7.4 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt From a8aedbeb7c20c7e2ec041e49d1567a499cd3acdb Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 29 Aug 2020 12:12:18 +0530 Subject: [PATCH 059/114] chore: rearrange imports --- scrapy/core/http2/protocol.py | 2 +- scrapy/core/http2/stream.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 0b872f6ba..e32e2b6fe 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -3,6 +3,7 @@ import itertools import logging from collections import deque from ipaddress import IPv4Address, IPv6Address +from typing import Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -21,7 +22,6 @@ from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin from twisted.python.failure import Failure from twisted.web.client import URI -from typing import Dict, List, Optional, Union from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index e01e76ae5..ef90773b6 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -2,6 +2,7 @@ import logging from enum import Enum from io import BytesIO from urllib.parse import urlparse +from typing import Dict, List, Optional, Tuple, TYPE_CHECKING from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError @@ -10,8 +11,6 @@ from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from typing import Dict -from typing import List, Optional, Tuple, TYPE_CHECKING from scrapy.http import Request from scrapy.http.headers import Headers From eff33a2e7950b29fd69adc40b17b7fe9b0e637c2 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 30 Aug 2020 23:54:43 +0530 Subject: [PATCH 060/114] fix(h2): Mockserver test uses H2DownloadHandler --- tests/test_downloader_handlers.py | 7 ++++--- tests/test_downloader_handlers_http2.py | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index e3777ee1d..c6f20cb50 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -601,6 +601,7 @@ class Https11CustomCiphers(unittest.TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" + settings_dict = None def setUp(self): self.mockserver = MockServer() @@ -611,7 +612,7 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download_with_content_length(self): - crawler = get_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # download it yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000})) @@ -620,7 +621,7 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download(self): - crawler = get_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) yield crawler.crawl(seed=Request(url=self.mockserver.url(''))) failure = crawler.spider.meta.get('failure') self.assertTrue(failure is None) @@ -629,7 +630,7 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download_gzip_response(self): - crawler = get_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) body = b'1' * 100 # PayloadResource requires body length to be 100 request = Request(self.mockserver.url('/payload'), method='POST', body=body, meta={'download_maxsize': 50}) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 583dc1d17..253646040 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -110,6 +110,11 @@ class Https2CustomCiphers(Https11CustomCiphers): class Http2MockServerTestCase(Http11MockServerTestCase): """HTTP 2.0 test case with MockServer""" + settings_dict = { + 'DOWNLOAD_HANDLERS': { + 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler' + } + } class Https2ProxyTestCase(Http11ProxyTestCase): From ddc26f3f8fdf766587eda55dfaf60b524eb89e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 1 Sep 2020 11:26:07 +0200 Subject: [PATCH 061/114] Revert Travis CI changes --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d9a8e512a..b883c5b78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ dist: xenial branches: only: - master - - http2 # ToDo: Remove once merged into master - /^\d\.\d+$/ - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: From 4d6359df2dfa3d561088faf097f6abe3f850a4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 11 Sep 2020 13:51:05 +0200 Subject: [PATCH 062/114] Mark HTTP/2 as experimental --- docs/topics/asyncio.rst | 11 ++++++----- docs/topics/commands.rst | 2 -- docs/topics/settings.rst | 4 ++++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index bfb430d52..c04044e8f 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -4,13 +4,14 @@ asyncio .. versionadded:: 2.0 -Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio -reactor `, you may use :mod:`asyncio` and +Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the +asyncio reactor `, you may use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy - versions may introduce related changes without a deprecation - period or warning. +.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet + recommended for production environments. Future Scrapy versions + may introduce related changes without a deprecation period or + warning. .. _install-asyncio: diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 7de5e8121..eef6b36ff 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -598,8 +598,6 @@ Example: Register commands via setup.py entry points ------------------------------------------- -.. note:: This is an experimental feature, use with caution. - You can also add Scrapy commands from an external library by adding a ``scrapy.commands`` section in the entry points of the library ``setup.py`` file. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8794e0259..218b23c87 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -678,6 +678,10 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update Scrapy currently does not support HTTP/2 Cleartext (h2c) since none of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_). +.. warning:: HTTP/2 support in Scrapy is experimental, and not yet recommended + for production environments. Future Scrapy versions may introduce + related changes without a deprecation period or warning. + .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. setting:: DOWNLOAD_TIMEOUT From 6e8d20a07a8c1a1f3ad7b3c3d74b7d37f2b47327 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Wed, 16 Sep 2020 04:57:07 -0300 Subject: [PATCH 063/114] HTTP/2: add some type hints (#4785) --- scrapy/core/downloader/handlers/http2.py | 24 +++++++++++++-------- scrapy/core/http2/agent.py | 27 ++++++++++++++---------- scrapy/core/http2/protocol.py | 15 ++++++++----- scrapy/core/http2/stream.py | 7 +++--- setup.cfg | 6 ------ tests/test_downloader_handlers.py | 19 +++++++++-------- tests/test_http2_client_protocol.py | 2 +- 7 files changed, 56 insertions(+), 44 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 4be888bda..e97c31e90 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,8 +1,9 @@ import warnings from time import time -from typing import Optional +from typing import Optional, Type, TypeVar from urllib.parse import urldefrag +from twisted.internet.base import DelayedCall from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI @@ -10,14 +11,18 @@ from twisted.web.client import URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent +from scrapy.crawler import Crawler from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes +H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler") + + class H2DownloadHandler: - def __init__(self, settings: Settings, crawler=None): + def __init__(self, settings: Settings, crawler: Optional[Crawler] = None): self._crawler = crawler from twisted.internet import reactor @@ -25,14 +30,14 @@ class H2DownloadHandler: self._context_factory = load_context_factory_from_settings(settings, crawler) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass: return cls(crawler.settings, crawler) def download_request(self, request: Request, spider: Spider) -> Deferred: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, - crawler=self._crawler + crawler=self._crawler, ) return agent.download_request(request, spider) @@ -47,8 +52,9 @@ class ScrapyH2Agent: def __init__( self, context_factory, pool: H2ConnectionPool, - connect_timeout=10, bind_address: Optional[bytes] = None, - crawler=None + connect_timeout: int = 10, + bind_address: Optional[bytes] = None, + crawler: Optional[Crawler] = None, ) -> None: self._context_factory = context_factory self._connect_timeout = connect_timeout @@ -80,7 +86,7 @@ class ScrapyH2Agent: proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')), connect_timeout=timeout, bind_address=bind_address, - pool=self._pool + pool=self._pool, ) return self._Agent( @@ -88,7 +94,7 @@ class ScrapyH2Agent: context_factory=self._context_factory, connect_timeout=timeout, bind_address=bind_address, - pool=self._pool + pool=self._pool, ) def download_request(self, request: Request, spider: Spider) -> Deferred: @@ -110,7 +116,7 @@ class ScrapyH2Agent: return response @staticmethod - def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl) -> Response: + def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response: if timeout_cl.active(): timeout_cl.cancel() return response diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index d950c6cfb..a142fa210 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,5 +1,5 @@ from collections import deque -from typing import Deque, Dict, List, Tuple, Optional +from typing import Deque, Dict, List, Optional, Tuple from twisted.internet import defer from twisted.internet.base import ReactorBase @@ -95,16 +95,18 @@ class H2ConnectionPool: class H2Agent: def __init__( - self, reactor: ReactorBase, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + self, + reactor: ReactorBase, + pool: H2ConnectionPool, + context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, + bind_address: Optional[bytes] = None, ) -> None: self._reactor = reactor self._pool = pool self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2']) self.endpoint_factory = _StandardEndpointFactory( - self._reactor, self._context_factory, - connect_timeout, bind_address + self._reactor, self._context_factory, connect_timeout, bind_address ) def get_endpoint(self, uri: URI): @@ -132,17 +134,20 @@ class H2Agent: class ScrapyProxyH2Agent(H2Agent): def __init__( - self, reactor: ReactorBase, - proxy_uri: URI, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + self, + reactor: ReactorBase, + proxy_uri: URI, + pool: H2ConnectionPool, + context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, + bind_address: Optional[bytes] = None, ) -> None: super(ScrapyProxyH2Agent, self).__init__( reactor=reactor, pool=pool, context_factory=context_factory, connect_timeout=connect_timeout, - bind_address=bind_address + bind_address=bind_address, ) self._proxy_uri = proxy_uri diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index e32e2b6fe..9d499596c 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -29,6 +29,7 @@ from scrapy.http import Request from scrapy.settings import Settings from scrapy.spiders import Spider + logger = logging.getLogger(__name__) @@ -42,7 +43,11 @@ class InvalidNegotiatedProtocol(H2Error): class RemoteTerminatedConnection(H2Error): - def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], event: ConnectionTerminated): + def __init__( + self, + remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], + event: ConnectionTerminated, + ) -> None: self.remote_ip_address = remote_ip_address self.terminate_event = event @@ -51,7 +56,7 @@ class RemoteTerminatedConnection(H2Error): class MethodNotAllowed405(H2Error): - def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]): + def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]) -> None: self.remote_ip_address = remote_ip_address def __str__(self) -> str: @@ -220,13 +225,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.conn.initiate_connection() self._write_to_transport() - def _lose_connection_with_error(self, errors: List[BaseException]): + def _lose_connection_with_error(self, errors: List[BaseException]) -> None: """Helper function to lose the connection with the error sent as a reason""" self._conn_lost_errors += errors self.transport.loseConnection() - def handshakeCompleted(self): + def handshakeCompleted(self) -> None: """We close the connection with InvalidNegotiatedProtocol exception when the connection was not made via h2 protocol""" negotiated_protocol = self.transport.negotiatedProtocol @@ -263,7 +268,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): finally: self._write_to_transport() - def timeoutConnection(self): + def timeoutConnection(self) -> None: """Called when the connection times out. We lose the connection with TimeoutError""" diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index ef90773b6..3ae2e8db8 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -19,6 +19,7 @@ from scrapy.responsetypes import responsetypes if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol + logger = logging.getLogger(__name__) @@ -27,7 +28,7 @@ class InactiveStreamClosed(ConnectionClosed): of the stream. This happens when a stream is waiting for other streams to close and connection is lost.""" - def __init__(self, request: Request): + def __init__(self, request: Request) -> None: self.request = request def __str__(self) -> str: @@ -139,7 +140,7 @@ class Stream: 'headers': Headers({}), } - def _cancel(_): + def _cancel(_) -> None: # Close this stream as gracefully as possible # If the associated request is initiated we reset this stream # else we directly call close() method @@ -360,7 +361,7 @@ class Stream: self, reason: StreamCloseReason, errors: Optional[List[BaseException]] = None, - from_protocol: bool = False + from_protocol: bool = False, ) -> None: """Based on the reason sent we will handle each case. """ diff --git a/setup.cfg b/setup.cfg index 94c1e7b89..8101443e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -82,9 +82,6 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True -[mypy-tests.test_downloader_handlers_http2] -ignore_errors = True - [mypy-tests.test_engine] ignore_errors = True @@ -94,9 +91,6 @@ ignore_errors = True [mypy-tests.test_http_request] ignore_errors = True -[mypy-tests.test_http2_client_protocol] -ignore_errors = True - [mypy-tests.test_linkextractors] ignore_errors = True diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 20e31f7f7..5b4a2d270 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -2,6 +2,7 @@ import contextlib import os import shutil import tempfile +from typing import Optional, Type from unittest import mock from testfixtures import LogCapture @@ -206,7 +207,7 @@ class LargeChunkedFileResource(resource.Resource): class HttpTestCase(unittest.TestCase): scheme = 'http' - download_handler_cls = HTTPDownloadHandler + download_handler_cls: Type = HTTPDownloadHandler # only used for HTTPS tests keyfile = 'keys/localhost.key' @@ -365,7 +366,7 @@ class HttpTestCase(unittest.TestCase): class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" - download_handler_cls = HTTP10DownloadHandler + download_handler_cls: Type = HTTP10DownloadHandler class Https10TestCase(Http10TestCase): @@ -374,7 +375,7 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler def test_download_without_maxsize_limit(self): request = Request(self.getURL('file')) @@ -561,7 +562,7 @@ class Https11InvalidDNSPattern(Https11TestCase): class Https11CustomCiphers(unittest.TestCase): scheme = 'https' - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler keyfile = 'keys/localhost.key' certfile = 'keys/localhost.crt' @@ -601,7 +602,7 @@ class Https11CustomCiphers(unittest.TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" - settings_dict = None + settings_dict: Optional[dict] = None def setUp(self): self.mockserver = MockServer() @@ -668,7 +669,7 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): - download_handler_cls = HTTPDownloadHandler + download_handler_cls: Type = HTTPDownloadHandler expected_http_proxy_request_body = b'http://example.com' def setUp(self): @@ -721,14 +722,14 @@ class HttpProxyTestCase(unittest.TestCase): class Http10ProxyTestCase(HttpProxyTestCase): - download_handler_cls = HTTP10DownloadHandler + download_handler_cls: Type = HTTP10DownloadHandler def test_download_with_proxy_https_noconnect(self): raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler') class Http11ProxyTestCase(HttpProxyTestCase): - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): @@ -776,7 +777,7 @@ class S3AnonTestCase(unittest.TestCase): class S3TestCase(unittest.TestCase): - download_handler_cls = S3DownloadHandler + download_handler_cls: Type = S3DownloadHandler # test use same example keys than amazon developer guide # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 4926ada14..d9ab553f0 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -45,7 +45,7 @@ def make_html_body(val): class DummySpider(Spider): name = 'dummy' - start_urls = [] + start_urls: list = [] def parse(self, response): print(response) From bde96a5ad98caed74cbaaeaf1dfe4b215093c03f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Nov 2020 16:42:44 +0100 Subject: [PATCH 064/114] Ignore server-initiated events --- scrapy/core/http2/protocol.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 9d499596c..67a86bd62 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -349,10 +349,16 @@ class H2ClientProtocol(Protocol, TimeoutMixin): ]) def data_received(self, event: DataReceived) -> None: - self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) + try: + self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) + except KeyError: + logger.debug(f'Ignoring server-initiated event {event}') def response_received(self, event: ResponseReceived) -> None: - self.streams[event.stream_id].receive_headers(event.headers) + try: + self.streams[event.stream_id].receive_headers(event.headers) + except KeyError: + logger.debug(f'Ignoring server-initiated event {event}') def settings_acknowledged(self, event: SettingsAcknowledged) -> None: self.metadata['settings_acknowledged'] = True @@ -365,12 +371,20 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) def stream_ended(self, event: StreamEnded) -> None: - stream = self.pop_stream(event.stream_id) - stream.close(StreamCloseReason.ENDED, from_protocol=True) + try: + stream = self.pop_stream(event.stream_id) + except KeyError: + logger.debug(f'Ignoring server-initiated event {event}') + else: + stream.close(StreamCloseReason.ENDED, from_protocol=True) def stream_reset(self, event: StreamReset) -> None: - stream = self.pop_stream(event.stream_id) - stream.close(StreamCloseReason.RESET, from_protocol=True) + try: + stream = self.pop_stream(event.stream_id) + except KeyError: + logger.debug(f'Ignoring server-initiated event {event}') + else: + stream.close(StreamCloseReason.RESET, from_protocol=True) def window_updated(self, event: WindowUpdated) -> None: if event.stream_id != 0: From 08f5ed712f4fdafec122f887c3ef06629f35ee0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Nov 2020 17:38:18 +0100 Subject: [PATCH 065/114] Fix memory issue due to unexpectedly large server frames --- scrapy/core/http2/protocol.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 67a86bd62..d8d0974b8 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -13,7 +13,7 @@ from h2.events import ( SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived, WindowUpdated ) -from h2.exceptions import H2Error +from h2.exceptions import FrameTooLargeError, H2Error from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory @@ -261,6 +261,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin): events = self.conn.receive_data(data) self._handle_events(events) except H2Error as e: + if isinstance(e, FrameTooLargeError): + # hyper-h2 does not drop the connection in this scenario, we + # need to abort the connection manually. + self._conn_lost_errors += [e] + self.transport.abortConnection() + return + # Save this error as ultimately the connection will be dropped # internally by hyper-h2. Saved error will be passed to all the streams # closed with the connection. From e494a3f73318db76d7c56e65bc632cd5875b8825 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 31 Dec 2020 11:50:15 -0300 Subject: [PATCH 066/114] protocol attribute for h2 responses --- docs/topics/request-response.rst | 2 +- scrapy/core/http2/stream.py | 2 ++ tests/test_downloader_handlers_http2.py | 7 +++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 98906992d..48f7f4a87 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -694,7 +694,7 @@ Response objects :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` :param protocol: The protocol that was used to download the response. - For instance: "HTTP/1.0", "HTTP/1.1" + For instance: "HTTP/1.0", "HTTP/1.1", "h2" :type protocol: :class:`str` .. versionadded:: 2.0.0 diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 3ae2e8db8..e345ca79a 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -15,6 +15,7 @@ from twisted.web.client import ResponseFailed from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes +from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -458,6 +459,7 @@ class Stream: request=self._request, certificate=self._protocol.metadata['certificate'], ip_address=self._protocol.metadata['ip_address'], + protocol=to_unicode(self._protocol.transport.negotiatedProtocol), ) self._deferred_response.callback(response) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 253646040..8f7f7aee0 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -22,6 +22,13 @@ class Https2TestCase(Https11TestCase): download_handler_cls = H2DownloadHandler HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "h2") + return d + @defer.inlineCallbacks def test_download_with_maxsize_very_large_file(self): with mock.patch('scrapy.core.http2.stream.logger') as logger: From 2ce8e0c74200481f8249bcee0f2a8a249e8fd58c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 09:09:53 +0100 Subject: [PATCH 067/114] Document the (hard-coded) maximum HTTP/2 frame size accepted from servers --- docs/topics/settings.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index b948dbfde..c7b59d582 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -689,10 +689,15 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update Scrapy currently does not support HTTP/2 Cleartext (h2c) since none of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_). + Also, Scrapy does not currently support specifying a maximum `frame size`_ + larger than the default value, 16384. Connections to servers that send a + larger frame will fail. + .. warning:: HTTP/2 support in Scrapy is experimental, and not yet recommended for production environments. Future Scrapy versions may introduce related changes without a deprecation period or warning. +.. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. setting:: DOWNLOAD_TIMEOUT From d1024566d85e52f71f5591e53a3a9ee4867148ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 09:13:45 +0100 Subject: [PATCH 068/114] =?UTF-8?q?setup.py:=20Twisted=20=E2=86=92=20Twist?= =?UTF-8?q?ed[http2]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a95014c98..d1d6cc4b6 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support(): install_requires = [ - 'Twisted>=17.9.0', + 'Twisted[http2]>=17.9.0', 'cryptography>=2.0', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', From 536e749eccbdf479c9849b1868a5d95aa1210225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 09:22:02 +0100 Subject: [PATCH 069/114] HTTP/2: remove verbose protocol-handling logging --- scrapy/core/http2/protocol.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index d8d0974b8..36a51b898 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -218,7 +218,6 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.setTimeout(self.IDLE_TIMEOUT) destination = self.transport.getPeer() - logger.debug('Connection made to {}'.format(destination)) self.metadata['ip_address'] = ipaddress.ip_address(destination.host) # Initiate H2 Connection @@ -347,7 +346,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) elif isinstance(event, UnknownFrameReceived): - logger.debug('UnknownFrameReceived: frame={}'.format(event.frame)) + logger.warning(f'Unknown frame received: {event.frame}') # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated) -> None: @@ -357,15 +356,19 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def data_received(self, event: DataReceived) -> None: try: - self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) + stream = self.streams[event.stream_id] except KeyError: - logger.debug(f'Ignoring server-initiated event {event}') + pass # We ignore server-initiated events + else: + stream.receive_data(event.data, event.flow_controlled_length) def response_received(self, event: ResponseReceived) -> None: try: - self.streams[event.stream_id].receive_headers(event.headers) + stream = self.streams[event.stream_id] except KeyError: - logger.debug(f'Ignoring server-initiated event {event}') + pass # We ignore server-initiated events + else: + stream.receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: self.metadata['settings_acknowledged'] = True @@ -381,7 +384,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): try: stream = self.pop_stream(event.stream_id) except KeyError: - logger.debug(f'Ignoring server-initiated event {event}') + pass # We ignore server-initiated events else: stream.close(StreamCloseReason.ENDED, from_protocol=True) @@ -389,7 +392,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): try: stream = self.pop_stream(event.stream_id) except KeyError: - logger.debug(f'Ignoring server-initiated event {event}') + pass # We ignore server-initiated events else: stream.close(StreamCloseReason.RESET, from_protocol=True) From 1a7bde0d8e142acc06a534f6daf8910fde5de06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 10:55:11 +0100 Subject: [PATCH 070/114] Document that HTTP/2 server pushes are ignored --- docs/topics/settings.rst | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c7b59d582..05cb4a855 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -684,21 +684,28 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', } +.. warning:: + + HTTP/2 support in Scrapy is experimental, and not yet recommended for + production environments. Future Scrapy versions may introduce related + changes without a deprecation period or warning. + .. note:: - Scrapy currently does not support HTTP/2 Cleartext (h2c) since none - of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_). + Known limitations of the current HTTP/2 implementation of Scrapy include: - Also, Scrapy does not currently support specifying a maximum `frame size`_ - larger than the default value, 16384. Connections to servers that send a - larger frame will fail. + - No support for HTTP/2 Cleartext (h2c), since no major browser supports + HTTP/2 unencrypted (refer `http2 faq`_). -.. warning:: HTTP/2 support in Scrapy is experimental, and not yet recommended - for production environments. Future Scrapy versions may introduce - related changes without a deprecation period or warning. + - No setting to specify a maximum `frame size`_ larger than the default + value, 16384. Connections to servers that send a larger frame will + fail. + + - No support for `server pushes`_, which are ignored. .. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption +.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2 .. setting:: DOWNLOAD_TIMEOUT From 4c801551fa10e4ff75f0768b903664f415a6a504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 21:11:46 +0100 Subject: [PATCH 071/114] Document that the bytes_received signal is not yet implemented for HTTP/2 --- docs/topics/settings.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 05cb4a855..0a4684a91 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -703,6 +703,8 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update - No support for `server pushes`_, which are ignored. + - No support for the :signal:`bytes_received` signal. + .. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2 From 248800328cb32f79a214289d59e927b576e21712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 21:13:43 +0100 Subject: [PATCH 072/114] Fix test_pinned_twisted_version --- tests/test_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 93e7311d2..5e63ebffb 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -36,7 +36,7 @@ class ScrapyUtilsTest(unittest.TestCase): ) config_parser = ConfigParser() config_parser.read(tox_config_file_path) - pattern = r'Twisted==([\d.]+)' + pattern = r'Twisted\[http2\]==([\d.]+)' match = re.search(pattern, config_parser['pinned']['deps']) pinned_twisted_version_string = match[1] From 0e4b291701baa8b74bd13bd47831dd040dd7173f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 21:28:04 +0100 Subject: [PATCH 073/114] HTTP/2: fix canceling a request before a connection has been established --- scrapy/core/http2/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index e345ca79a..572dbf7aa 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -459,7 +459,7 @@ class Stream: request=self._request, certificate=self._protocol.metadata['certificate'], ip_address=self._protocol.metadata['ip_address'], - protocol=to_unicode(self._protocol.transport.negotiatedProtocol), + protocol='h2', ) self._deferred_response.callback(response) From 7b11b74c77d1535f943baebbf5c794a63d147a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 4 Feb 2021 11:08:01 +0100 Subject: [PATCH 074/114] Use --use-deprecated=legacy-resolver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let’s see how test results change --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index d8e900e06..ecd3aad6e 100644 --- a/tox.ini +++ b/tox.ini @@ -27,6 +27,10 @@ passenv = GCS_PROJECT_ID #allow tox virtualenv to upgrade pip/wheel/setuptools download = true +# TODO: Remove the custom install_command below +# Temporary workaround to filter out errors caused by the insanely long time +# that it takes for the new resolver to install dependencies. +install_command=python -m pip install --use-deprecated=legacy-resolver {opts} {packages} commands = py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} From 7afcd634ab7d71a465cd9406091d371dca105789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 5 Feb 2021 13:04:54 +0100 Subject: [PATCH 075/114] Remove unused import --- scrapy/core/http2/stream.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 572dbf7aa..8a1b3e470 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -15,7 +15,6 @@ from twisted.web.client import ResponseFailed from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes -from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol From 8527b53e14e729504f6e382dd78ef6b0457fc7e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 5 Feb 2021 13:06:27 +0100 Subject: [PATCH 076/114] Revert "Use --use-deprecated=legacy-resolver" This reverts commit 7b11b74c77d1535f943baebbf5c794a63d147a13. --- tox.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tox.ini b/tox.ini index ecd3aad6e..d8e900e06 100644 --- a/tox.ini +++ b/tox.ini @@ -27,10 +27,6 @@ passenv = GCS_PROJECT_ID #allow tox virtualenv to upgrade pip/wheel/setuptools download = true -# TODO: Remove the custom install_command below -# Temporary workaround to filter out errors caused by the insanely long time -# that it takes for the new resolver to install dependencies. -install_command=python -m pip install --use-deprecated=legacy-resolver {opts} {packages} commands = py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} From 45345ba6b508fae426223039e491dd721eaac9a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 8 Feb 2021 17:56:29 +0100 Subject: [PATCH 077/114] Use constraints.txt to limit pip resolver backtracking --- tests/constraints.txt | 9 ++++++++- tox.ini | 1 - 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/constraints.txt b/tests/constraints.txt index 5655ac2d3..3b30e6bb5 100644 --- a/tests/constraints.txt +++ b/tests/constraints.txt @@ -1 +1,8 @@ -Twisted!=18.4.0 \ No newline at end of file +# Request the latest known version or newer of some dependencies to prevent the +# pip dependency resolver from spending too much time backtracking. +attrs>=20.2.0 +Pillow>=8.0.1 +pytest>=6.2.1 +pytest-twisted>=1.13.1 +sybil>=2.0.0 +Twisted>=19.10.0 diff --git a/tox.ini b/tox.ini index d8e900e06..9908a4d51 100644 --- a/tox.ini +++ b/tox.ini @@ -67,7 +67,6 @@ commands = [pinned] deps = - -ctests/constraints.txt cryptography==2.0 cssselect==0.9.1 h2==3.2.0 From bb72bba1786bc6a725df24a01dffba2e6e2cb7c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 8 Feb 2021 21:51:57 +0100 Subject: [PATCH 078/114] tox: apply upper constraints to all non-pinned package installations --- tests/{constraints.txt => upper-constraints.txt} | 8 ++++++++ tox.ini | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) rename tests/{constraints.txt => upper-constraints.txt} (64%) diff --git a/tests/constraints.txt b/tests/upper-constraints.txt similarity index 64% rename from tests/constraints.txt rename to tests/upper-constraints.txt index 3b30e6bb5..c8c57deea 100644 --- a/tests/constraints.txt +++ b/tests/upper-constraints.txt @@ -1,8 +1,16 @@ # Request the latest known version or newer of some dependencies to prevent the # pip dependency resolver from spending too much time backtracking. attrs>=20.2.0 +Automat>=0.8.0 +itemadapter>=0.1.1 +itemloaders>=1.0.3 +lxml>=4.6.1 +parsel>=1.5.2 Pillow>=8.0.1 +pyOpenSSL>=20.0.0 pytest>=6.2.1 pytest-twisted>=1.13.1 +service_identity>=17.0.0 +six>=1.14.0 sybil>=2.0.0 Twisted>=19.10.0 diff --git a/tox.ini b/tox.ini index 9908a4d51..2dfe8987c 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,6 @@ minversion = 1.7.0 [testenv] deps = - -ctests/constraints.txt -rtests/requirements-py3.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 @@ -29,6 +28,8 @@ passenv = download = true commands = py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} +install_command = + pip install -U -ctests/upper-constraints.txt {opts} {packages} [testenv:typing] basepython = python3 @@ -90,12 +91,15 @@ deps = Pillow==4.0.0 setenv = _SCRAPY_PINNED=true +install_command = + pip install -U {opts} {packages} [testenv:pinned] deps = {[pinned]deps} lxml==3.5.0 PyDispatcher==2.0.5 +install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -107,6 +111,7 @@ deps = # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 PyDispatcher==2.0.5 +install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -123,6 +128,7 @@ commands = [testenv:asyncio-pinned] deps = {[testenv:pinned]deps} commands = {[testenv:asyncio]commands} +install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -138,6 +144,7 @@ deps = lxml==4.0.0 PyPyDispatcher==2.1.0 commands = {[testenv:pypy3]commands} +install_command = {[pinned]install_command} setenv = {[pinned]setenv} From 9ac5b1d021562a20b2a7d437f8bacb6754426811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 8 Feb 2021 22:31:20 +0100 Subject: [PATCH 079/114] Adjust test constraints --- tests/upper-constraints.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/upper-constraints.txt b/tests/upper-constraints.txt index c8c57deea..75f337856 100644 --- a/tests/upper-constraints.txt +++ b/tests/upper-constraints.txt @@ -2,12 +2,13 @@ # pip dependency resolver from spending too much time backtracking. attrs>=20.2.0 Automat>=0.8.0 +botocore>=1.20.3 itemadapter>=0.1.1 itemloaders>=1.0.3 lxml>=4.6.1 parsel>=1.5.2 Pillow>=8.0.1 -pyOpenSSL>=20.0.0 +pyOpenSSL>=17.5 # mitmproxy 4.0.4 pytest>=6.2.1 pytest-twisted>=1.13.1 service_identity>=17.0.0 From 15b501c0898150de686ee42dd4d78a411891e795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 10 Feb 2021 18:10:57 +0100 Subject: [PATCH 080/114] Do not force string interpolation while logging --- scrapy/core/http2/protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 36a51b898..968bfce63 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -346,7 +346,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) elif isinstance(event, UnknownFrameReceived): - logger.warning(f'Unknown frame received: {event.frame}') + logger.warning('Unknown frame received: %s', event.frame) # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated) -> None: From e80f37bd3fa0f2262e898f712541bb7a12e89110 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 17 Feb 2021 16:34:29 -0300 Subject: [PATCH 081/114] Test http2 agent for unsupported scheme --- tests/test_downloader_handlers_http2.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 8f7f7aee0..bee9ae75b 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -3,6 +3,7 @@ from unittest import mock from twisted.internet import defer, error, reactor from twisted.trial import unittest from twisted.web import server +from twisted.web.error import SchemeNotSupported from scrapy.core.downloader.handlers.http2 import H2DownloadHandler from scrapy.http import Request @@ -48,6 +49,12 @@ class Https2TestCase(Https11TestCase): reactor.callLater(.1, d.callback, logger) yield d + @defer.inlineCallbacks + def test_unsupported_scheme(self): + request = Request("ftp://unsupported.scheme") + d = self.download_request(request, Spider("foo")) + yield self.assertFailure(d, SchemeNotSupported) + def test_download_broken_content_cause_data_loss(self, url='broken'): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) From 4418f78941fc9d89ada9ad0436ebd0ae4ece1017 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 17 Feb 2021 18:36:52 -0300 Subject: [PATCH 082/114] Simplify check for negotiated protocol negotiatedProtocol's type is Optional[bytes] See https://github.com/twisted/twisted/blob/twisted-20.3.0/src/twisted/protocols/tls.py#L563-L587 and https://www.pyopenssl.org/en/20.0.1/api/ssl.html#OpenSSL.SSL.Connection.get_alpn_proto_negotiated Note that OpenSSL.SSL.Connection.get_next_proto_negotiated is deprecated: https://www.pyopenssl.org/en/20.0.0/changelog.html#backward-incompatible-changes --- scrapy/core/http2/protocol.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 968bfce63..9d7da14c1 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -233,13 +233,11 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def handshakeCompleted(self) -> None: """We close the connection with InvalidNegotiatedProtocol exception when the connection was not made via h2 protocol""" - negotiated_protocol = self.transport.negotiatedProtocol - if isinstance(negotiated_protocol, bytes): - negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') - if negotiated_protocol != 'h2': + protocol = self.transport.negotiatedProtocol + if protocol is not None and protocol != b"h2": # Here we have not initiated the connection yet # So, no need to send a GOAWAY frame to the remote - self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) + self._lose_connection_with_error([InvalidNegotiatedProtocol(protocol.decode("utf-8"))]) def _check_received_data(self, data: bytes) -> None: """Checks for edge cases where the connection to remote fails From 6326178bc5824e8c08d84b6543de6977a16fb8d2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 22 Feb 2021 12:50:51 -0300 Subject: [PATCH 083/114] http2: acceptable protocol update, tests (#4994) --- scrapy/core/http2/protocol.py | 26 ++++++++++++++------------ tests/test_http2_client_protocol.py | 8 ++++++++ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 9d7da14c1..6ca69b23b 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -33,13 +33,16 @@ from scrapy.spiders import Spider logger = logging.getLogger(__name__) +PROTOCOL_NAME = b"h2" + + class InvalidNegotiatedProtocol(H2Error): - def __init__(self, negotiated_protocol: str) -> None: + def __init__(self, negotiated_protocol: bytes) -> None: self.negotiated_protocol = negotiated_protocol def __str__(self) -> str: - return f'InvalidNegotiatedProtocol: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}' + return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}") class RemoteTerminatedConnection(H2Error): @@ -52,7 +55,7 @@ class RemoteTerminatedConnection(H2Error): self.terminate_event = event def __str__(self) -> str: - return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address!r}' + return f'Received GOAWAY frame from {self.remote_ip_address!r}' class MethodNotAllowed405(H2Error): @@ -60,7 +63,7 @@ class MethodNotAllowed405(H2Error): self.remote_ip_address = remote_ip_address def __str__(self) -> str: - return f"MethodNotAllowed405: Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}" + return f"Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}" @implementer(IHandshakeListener) @@ -231,13 +234,12 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.transport.loseConnection() def handshakeCompleted(self) -> None: - """We close the connection with InvalidNegotiatedProtocol exception - when the connection was not made via h2 protocol""" - protocol = self.transport.negotiatedProtocol - if protocol is not None and protocol != b"h2": - # Here we have not initiated the connection yet - # So, no need to send a GOAWAY frame to the remote - self._lose_connection_with_error([InvalidNegotiatedProtocol(protocol.decode("utf-8"))]) + """ + Close the connection if it's not made via the expected protocol + """ + if self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME: + # we have not initiated the connection yet, no need to send a GOAWAY frame to the remote peer + self._lose_connection_with_error([InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)]) def _check_received_data(self, data: bytes) -> None: """Checks for edge cases where the connection to remote fails @@ -414,4 +416,4 @@ class H2ClientFactory(Factory): return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) def acceptableProtocols(self) -> List[bytes]: - return [b'h2'] + return [PROTOCOL_NAME] diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index d9ab553f0..8b2f6a11d 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -5,6 +5,7 @@ import re import shutil import string from ipaddress import IPv4Address +from unittest import mock from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError @@ -381,6 +382,13 @@ class Https2ClientProtocolTestCase(TestCase): 200 ) + @inlineCallbacks + def test_invalid_negotiated_protocol(self): + with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"): + request = Request(url=self.get_url('/status?n=200')) + with self.assertRaises(ResponseFailed): + yield self.make_request(request) + def test_cancel_request(self): request = Request(url=self.get_url('/get-data-html-large')) From 7605f19ec429fd3adb6a18da8e48143f99f6bfec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 23 Feb 2021 05:54:48 +0100 Subject: [PATCH 084/114] HTTP/2: test 2 concurrent requests to the same domain --- tests/test_downloader_handlers_http2.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index bee9ae75b..44d45b7d8 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -73,6 +73,21 @@ class Https2TestCase(Https11TestCase): def test_download_broken_chunked_content_allow_data_loss_via_setting(self): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + def test_concurrent_requests_same_domain(self): + spider = Spider('foo') + + request1 = Request(self.getURL('file')) + d1 = self.download_request(request1, spider) + d1.addCallback(lambda r: r.body) + d1.addCallback(self.assertEqual, b"0123456789") + + request2 = Request(self.getURL('echo'), method='POST') + d2 = self.download_request(request2, spider) + d2.addCallback(lambda r: r.headers['Content-Length']) + d2.addCallback(self.assertEqual, b"79") + + return defer.DeferredList([d1, d2]) + class Https2WrongHostnameTestCase(Https2TestCase): tls_log_message = ( From bd29f32dee445c77e7f2427a3047b7c74505efb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 23 Feb 2021 06:42:28 +0100 Subject: [PATCH 085/114] HTTP/2: do not make conn_lost_deferred optional --- scrapy/core/http2/protocol.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 6ca69b23b..1d150b7ce 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -70,7 +70,7 @@ class MethodNotAllowed405(H2Error): class H2ClientProtocol(Protocol, TimeoutMixin): IDLE_TIMEOUT = 240 - def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None: """ Arguments: uri -- URI of the base url to which HTTP/2 Connection will be made. @@ -308,8 +308,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): if not reason.check(connectionDone): self._conn_lost_errors.append(reason) - if self._conn_lost_deferred: - self._conn_lost_deferred.callback(self._conn_lost_errors) + self._conn_lost_deferred.callback(self._conn_lost_errors) for stream in self.streams.values(): if stream.metadata['request_sent']: @@ -407,7 +406,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): @implementer(IProtocolNegotiationFactory) class H2ClientFactory(Factory): - def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None: self.uri = uri self.settings = settings self.conn_lost_deferred = conn_lost_deferred From 5ba31cd1a268475db1f3cc64d4e6febb477e8f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 23 Feb 2021 11:57:33 +0100 Subject: [PATCH 086/114] HTTP/2 stream close reason handling: Use else + assert instead of elif --- scrapy/core/http2/stream.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 8a1b3e470..aa44c08ce 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -431,7 +431,8 @@ class Stream: errors.insert(0, InactiveStreamClosed(self._request)) self._deferred_response.errback(ResponseFailed(errors)) - elif reason is StreamCloseReason.INVALID_HOSTNAME: + else: + assert reason is StreamCloseReason.INVALID_HOSTNAME self._deferred_response.errback(InvalidHostname( self._request, str(self._protocol.metadata['uri'].host, 'utf-8'), From 510109420733e93db0d525302555224a8364ed5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 24 Feb 2021 07:33:39 +0100 Subject: [PATCH 087/114] HTTP/2: test a CONNECT request --- tests/test_downloader_handlers_http2.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 44d45b7d8..b5a40468a 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -1,5 +1,6 @@ from unittest import mock +from pytest import mark from twisted.internet import defer, error, reactor from twisted.trial import unittest from twisted.web import server @@ -88,6 +89,14 @@ class Https2TestCase(Https11TestCase): return defer.DeferredList([d1, d2]) + @mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") + def test_connect_request(self): + request = Request(self.getURL('file'), method='CONNECT') + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b'') + return d + class Https2WrongHostnameTestCase(Https2TestCase): tls_log_message = ( From a36f952198101bfdfeadf4bec079db120809dbb1 Mon Sep 17 00:00:00 2001 From: Wehzie <39304339+Wehzie@users.noreply.github.com> Date: Wed, 24 Feb 2021 08:15:44 +0100 Subject: [PATCH 088/114] fixed typo "an quotes.json" -> "a quotes.json" (#5005) --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 9270ff42c..740e47d0c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -464,7 +464,7 @@ The simplest way to store the scraped data is by using :ref:`Feed exports scrapy crawl quotes -O quotes.json -That will generate an ``quotes.json`` file containing all scraped items, +That will generate a ``quotes.json`` file containing all scraped items, serialized in `JSON`_. The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead From 12064d799b8f15eef770a615237932b880b594a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 24 Feb 2021 10:37:38 +0100 Subject: [PATCH 089/114] HTTP/2: improve header handling --- scrapy/core/http2/stream.py | 21 ++++++++--- tests/test_downloader_handlers_http2.py | 46 +++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index aa44c08ce..8a701e7c6 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -220,11 +220,24 @@ class Stream: (':path', path), ] - for name, value in self._request.headers.items(): - headers.append((str(name, 'utf-8'), str(value[0], 'utf-8'))) + content_length = str(len(self._request.body)) + headers.append(('Content-Length', content_length)) - if b'Content-Length' not in self._request.headers.keys(): - headers.append(('Content-Length', str(len(self._request.body)))) + content_length_name = self._request.headers.normkey(b'Content-Length') + for name, values in self._request.headers.items(): + for value in values: + value = str(value, 'utf-8') + if name == content_length_name: + if value != content_length: + logger.warning( + 'Ignoring bad Content-Length header %r of request %r, ' + 'sending %r instead', + value, + self._request, + content_length, + ) + continue + headers.append((str(name, 'utf-8'), value)) return headers diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index b5a40468a..7c3db5835 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -1,6 +1,8 @@ +import json from unittest import mock from pytest import mark +from testfixtures import LogCapture from twisted.internet import defer, error, reactor from twisted.trial import unittest from twisted.web import server @@ -97,6 +99,50 @@ class Https2TestCase(Https11TestCase): d.addCallback(self.assertEqual, b'') return d + def test_custom_content_length_good(self): + request = Request(self.getURL('contentlength')) + custom_content_length = str(len(request.body)) + request.headers['Content-Length'] = custom_content_length + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.text) + d.addCallback(self.assertEqual, custom_content_length) + return d + + def test_custom_content_length_bad(self): + request = Request(self.getURL('contentlength')) + actual_content_length = str(len(request.body)) + bad_content_length = str(len(request.body)+1) + request.headers['Content-Length'] = bad_content_length + log = LogCapture() + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.text) + d.addCallback(self.assertEqual, actual_content_length) + d.addCallback( + lambda _: log.check_present( + ( + 'scrapy.core.http2.stream', + 'WARNING', + f'Ignoring bad Content-Length header ' + f'{bad_content_length!r} of request {request}, sending ' + f'{actual_content_length!r} instead', + ) + ) + ) + d.addCallback( + lambda _: log.uninstall() + ) + return d + + def test_duplicate_header(self): + request = Request(self.getURL('echo')) + header, value1, value2 = 'Custom-Header', 'foo', 'bar' + request.headers.appendlist(header, value1) + request.headers.appendlist(header, value2) + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: json.loads(r.text)['headers'][header]) + d.addCallback(self.assertEqual, [value1, value2]) + return d + class Https2WrongHostnameTestCase(Https2TestCase): tls_log_message = ( From 386e2a51ae4ed7bd53374a5cadfdf380b58284ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 24 Feb 2021 10:41:01 +0100 Subject: [PATCH 090/114] tests/test_downloader_handlers_http2.py: fix style issue --- tests/test_downloader_handlers_http2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 7c3db5835..439778014 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -111,7 +111,7 @@ class Https2TestCase(Https11TestCase): def test_custom_content_length_bad(self): request = Request(self.getURL('contentlength')) actual_content_length = str(len(request.body)) - bad_content_length = str(len(request.body)+1) + bad_content_length = str(len(request.body) + 1) request.headers['Content-Length'] = bad_content_length log = LogCapture() d = self.download_request(request, Spider('foo')) From 3894ebb1497b32959c405201f2e010292cf65098 Mon Sep 17 00:00:00 2001 From: Djiar Date: Tue, 23 Feb 2021 15:34:53 +0100 Subject: [PATCH 091/114] Refactor curl_to_request_kwargs #5001 Co-authored-by: alkazaz alkazaz@kth.se Co-authored-by: swill swill@kth.se Co-authored-by: lerjevik lerjevik@kth.se Co-authored-by: aljica aljica@kth.se --- scrapy/utils/curl.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 6660b9dc0..d8b3deaa1 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -34,6 +34,26 @@ for argument in safe_to_ignore_arguments: curl_parser.add_argument(*argument, action='store_true') +def _parse_headers_and_cookies(parsed_args): + headers = [] + cookies = {} + for header in parsed_args.headers or (): + name, val = header.split(':', 1) + name = name.strip() + val = val.strip() + if name.title() == 'Cookie': + for name, morsel in SimpleCookie(val).items(): + cookies[name] = morsel.value + else: + headers.append((name, val)) + + if parsed_args.auth: + user, password = parsed_args.auth.split(':', 1) + headers.append(('Authorization', basic_auth_header(user, password))) + + return headers, cookies + + def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): """Convert a cURL command syntax to Request kwargs. @@ -70,21 +90,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): result = {'method': method.upper(), 'url': url} - headers = [] - cookies = {} - for header in parsed_args.headers or (): - name, val = header.split(':', 1) - name = name.strip() - val = val.strip() - if name.title() == 'Cookie': - for name, morsel in SimpleCookie(val).items(): - cookies[name] = morsel.value - else: - headers.append((name, val)) - - if parsed_args.auth: - user, password = parsed_args.auth.split(':', 1) - headers.append(('Authorization', basic_auth_header(user, password))) + headers, cookies = _parse_headers_and_cookies(parsed_args) if headers: result['headers'] = headers From f689615e8d61f1f651b869b7a6284ca6ca15cde7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 24 Feb 2021 12:54:56 +0100 Subject: [PATCH 092/114] Close files in the PerYearXmlExportPipeline documentation example --- docs/topics/exporters.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 0a0a1765a..8648daded 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -50,18 +50,19 @@ value of one of their fields:: self.year_to_exporter = {} def close_spider(self, spider): - for exporter in self.year_to_exporter.values(): + for exporter, xml_file in self.year_to_exporter.values(): exporter.finish_exporting() + xml_file.close() def _exporter_for_item(self, item): adapter = ItemAdapter(item) year = adapter['year'] if year not in self.year_to_exporter: - f = open(f'{year}.xml', 'wb') - exporter = XmlItemExporter(f) + xml_file = open(f'{year}.xml', 'wb') + exporter = XmlItemExporter(xml_file) exporter.start_exporting() - self.year_to_exporter[year] = exporter - return self.year_to_exporter[year] + self.year_to_exporter[year] = (exporter, xml_file) + return self.year_to_exporter[year][0] def process_item(self, item, spider): exporter = self._exporter_for_item(item) From 7a54580679f192c4f1b66775aaddf1bee1efe448 Mon Sep 17 00:00:00 2001 From: deepang17 <47976918+deepang17@users.noreply.github.com> Date: Sun, 28 Feb 2021 15:33:09 +0530 Subject: [PATCH 093/114] DOCS:Cover scrapy-bench in the documentation --- docs/topics/benchmarking.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index b01a66188..4e53900ee 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -81,5 +81,4 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -In the future, more cases will be added to the benchmarking suite to cover -other common scenarios. +To use it as a project for more complex Scrapy benchmarking: https://github.com/scrapy/scrapy-bench From b25616d107d88bb195f19dca4c7e886a9ba652d3 Mon Sep 17 00:00:00 2001 From: deepang17 <47976918+deepang17@users.noreply.github.com> Date: Sun, 28 Feb 2021 16:26:46 +0530 Subject: [PATCH 094/114] DOCS: Cover scrapy-bench in the documentation --- docs/topics/benchmarking.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 4e53900ee..b15836771 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -81,4 +81,5 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -To use it as a project for more complex Scrapy benchmarking: https://github.com/scrapy/scrapy-bench +To use it as a project for more complex Scrapy benchmarking: +https://github.com/scrapy/scrapy-bench From 36f1dbf665abd8935c3adda9a5abfef959573cdb Mon Sep 17 00:00:00 2001 From: deepang17 <47976918+deepang17@users.noreply.github.com> Date: Tue, 2 Mar 2021 22:12:44 +0530 Subject: [PATCH 095/114] DOCS: Covered scrapy-bench --- .vscode/settings.json | 3 +++ docs/topics/benchmarking.rst | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..500bc7007 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.linting.pylintEnabled": true +} \ No newline at end of file diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index b15836771..3e671365b 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -81,5 +81,6 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -To use it as a project for more complex Scrapy benchmarking: -https://github.com/scrapy/scrapy-bench +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench + +Use scrapy-bench_ for more complex benchmarking. \ No newline at end of file From 4fe26ae9701c95a05723a79649314da03e3ddc1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 2 Mar 2021 19:46:34 +0100 Subject: [PATCH 096/114] Limit tests to Twisted < 21 --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index e70aef2d2..69f52bd9f 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,8 @@ deps = # Extras botocore>=1.4.87 Pillow>=4.0.0 + # Twisted 21+ causes issues in tests that use skipIf + Twisted<21 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From 3d88ac605b04f4c60ca81ce509c0c3b25cfb8cd7 Mon Sep 17 00:00:00 2001 From: deepang17 <47976918+deepang17@users.noreply.github.com> Date: Tue, 9 Mar 2021 17:19:34 +0530 Subject: [PATCH 097/114] FIX: Updated benchmarking.rst --- .vscode/settings.json | 3 --- docs/topics/benchmarking.rst | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 500bc7007..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.linting.pylintEnabled": true -} \ No newline at end of file diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 3e671365b..0643df6a6 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -81,6 +81,6 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -.. _scrapy-bench: https://github.com/scrapy/scrapy-bench +Use scrapy-bench_ for more complex benchmarking. -Use scrapy-bench_ for more complex benchmarking. \ No newline at end of file +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench \ No newline at end of file From 3bea5e1a974b7cd99f75edb4ff728a9d7163a805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 9 Mar 2021 16:19:51 +0100 Subject: [PATCH 098/114] Remove unused _is_data_lost method --- scrapy/core/http2/stream.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 8a701e7c6..c2a4b702f 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -362,14 +362,6 @@ class Stream: self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) self.close(reason) - def _is_data_lost(self) -> bool: - assert self.metadata['stream_closed_server'] - - expected_size = self._response['flow_controlled_size'] - received_body_size = int(self._response['headers'][b'Content-Length']) - - return expected_size != received_body_size - def close( self, reason: StreamCloseReason, From 0c160882306a8e33f92815cc12dc8979f04f8f5e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 11 Mar 2021 11:52:35 -0300 Subject: [PATCH 099/114] headers_received signal (#4897) --- docs/faq.rst | 11 +-- docs/topics/exceptions.rst | 7 +- docs/topics/request-response.rst | 6 +- docs/topics/signals.rst | 32 +++++++- scrapy/core/downloader/handlers/http11.py | 24 ++++++ scrapy/signals.py | 1 + tests/spiders.py | 29 ++++++++ tests/test_crawl.py | 26 ++++++- tests/test_engine.py | 85 ++++++---------------- tests/test_engine_stop_download_bytes.py | 60 +++++++++++++++ tests/test_engine_stop_download_headers.py | 56 ++++++++++++++ 11 files changed, 260 insertions(+), 77 deletions(-) create mode 100644 tests/test_engine_stop_download_bytes.py create mode 100644 tests/test_engine_stop_download_headers.py diff --git a/docs/faq.rst b/docs/faq.rst index f492dfa30..9709885f6 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -398,11 +398,12 @@ How can I cancel the download of a given response? -------------------------------------------------- In some situations, it might be useful to stop the download of a certain response. -For instance, if you only need the first part of a large response and you would like -to save resources by avoiding the download of the whole body. -In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received` -signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to -the :ref:`topics-stop-response-download` topic for additional information and examples. +For instance, sometimes you can determine whether or not you need the full contents +of a response by inspecting its headers or the first bytes of its body. In that case, +you could save resources by attaching a handler to the :class:`~scrapy.signals.bytes_received` +or :class:`~scrapy.signals.headers_received` signals and raising a +:exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the +:ref:`topics-stop-response-download` topic for additional information and examples. .. _has been reported: https://github.com/scrapy/scrapy/issues/2905 diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 583a50ab8..2f1517906 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -85,8 +85,8 @@ StopDownload .. exception:: StopDownload(fail=True) -Raised from a :class:`~scrapy.signals.bytes_received` signal handler to -indicate that no further bytes should be downloaded for a response. +Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received` +signal handler to indicate that no further bytes should be downloaded for a response. The ``fail`` boolean parameter controls which method will handle the resulting response: @@ -110,5 +110,6 @@ attribute. ``StopDownload(False)`` or ``StopDownload(True)`` will raise a :class:`TypeError`. -See the documentation for the :class:`~scrapy.signals.bytes_received` signal +See the documentation for the :class:`~scrapy.signals.bytes_received` and +:class:`~scrapy.signals.headers_received` signals and the :ref:`topics-stop-response-download` topic for additional information and examples. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 98906992d..37008f3e9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -432,9 +432,9 @@ The meta key is used set retry times per request. When initialized, the Stopping the download of a Response =================================== -Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a -:class:`~scrapy.signals.bytes_received` signal handler will stop the -download of a given response. See the following example:: +Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a handler for the +:class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received` +signals will stop the download of a given response. See the following example:: import scrapy diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 1d99d8c28..98cfa606c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -384,6 +384,11 @@ bytes_received a possible scenario for a 25 kb response would be two signals fired with 10 kb of data, and a final one with 5 kb of data. + Handlers for this signal can stop the download of a response while it + is in progress by raising the :exc:`~scrapy.exceptions.StopDownload` + exception. Please refer to the :ref:`topics-stop-response-download` topic + for additional information and examples. + This signal does not support returning deferreds from its handlers. :param data: the data received by the download handler @@ -395,11 +400,36 @@ bytes_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.spiders.Spider` object -.. note:: Handlers of this signal can stop the download of a response while it +headers_received +~~~~~~~~~~~~~~~~ + +.. versionadded:: VERSION + +.. signal:: headers_received +.. function:: headers_received(headers, request, spider) + + Sent by the HTTP 1.1 and S3 download handlers when the response headers are + available for a given request, before downloading any additional content. + + Handlers for this signal can stop the download of a response while it is in progress by raising the :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the :ref:`topics-stop-response-download` topic for additional information and examples. + This signal does not support returning deferreds from its handlers. + + :param headers: the headers received by the download handler + :type headers: :class:`scrapy.http.headers.Headers` object + + :param body_length: expected size of the response body, in bytes + :type body_length: `int` + + :param request: the request that generated the download + :type request: :class:`~scrapy.http.Request` object + + :param spider: the spider associated with the response + :type spider: :class:`~scrapy.spiders.Spider` object + Response signals ---------------- diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 513df2de9..516a4326b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -382,6 +382,29 @@ class ScrapyAgent: return result def _cb_bodyready(self, txresponse, request): + headers_received_result = self._crawler.signals.send_catch_log( + signal=signals.headers_received, + headers=Headers(txresponse.headers.getAllRawHeaders()), + body_length=txresponse.length, + request=request, + spider=self._crawler.spider, + ) + for handler, result in headers_received_result: + if isinstance(result, Failure) and isinstance(result.value, StopDownload): + logger.debug("Download stopped for %(request)s from signal handler %(handler)s", + {"request": request, "handler": handler.__qualname__}) + txresponse._transport.stopProducing() + with suppress(AttributeError): + txresponse._transport._producer.loseConnection() + return { + "txresponse": txresponse, + "body": b"", + "flags": ["download_stopped"], + "certificate": None, + "ip_address": None, + "failure": result if result.value.fail else None, + } + # deliverBody hangs for responses without body if txresponse.length == 0: return { @@ -529,6 +552,7 @@ class _ResponseReader(protocol.Protocol): if isinstance(result, Failure) and isinstance(result.value, StopDownload): logger.debug("Download stopped for %(request)s from signal handler %(handler)s", {"request": self._request, "handler": handler.__qualname__}) + self.transport.stopProducing() self.transport._producer.loseConnection() failure = result if result.value.fail else None self._finish_response(flags=["download_stopped"], failure=failure) diff --git a/scrapy/signals.py b/scrapy/signals.py index c61ae6ec3..8cf2a4d93 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -17,6 +17,7 @@ request_reached_downloader = object() request_left_downloader = object() response_received = object() response_downloaded = object() +headers_received = object() bytes_received = object() item_scraped = object() item_dropped = object() diff --git a/tests/spiders.py b/tests/spiders.py index 106392ea6..7e579098a 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -390,3 +390,32 @@ class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider): def bytes_received(self, data, request, spider): self.meta["bytes_received"] = data raise StopDownload(fail=True) + + +class HeadersReceivedCallbackSpider(MetaSpider): + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + crawler.signals.connect(spider.headers_received, signals.headers_received) + return spider + + def start_requests(self): + yield Request(self.mockserver.url("/status"), errback=self.errback) + + def parse(self, response): + self.meta["response"] = response + + def errback(self, failure): + self.meta["failure"] = failure + + def headers_received(self, headers, body_length, request, spider): + self.meta["headers_received"] = headers + raise StopDownload(fail=False) + + +class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider): + + def headers_received(self, headers, body_length, request, spider): + self.meta["headers_received"] = headers + raise StopDownload(fail=True) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1083c1678..84bac9b50 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -35,6 +35,8 @@ from tests.spiders import ( DelaySpider, DuplicateStartRequestsSpider, FollowAllSpider, + HeadersReceivedCallbackSpider, + HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, ) @@ -496,7 +498,7 @@ class CrawlSpiderTestCase(TestCase): self.assertEqual(str(ip_address), gethostbyname(expected_netloc)) @defer.inlineCallbacks - def test_stop_download_callback(self): + def test_bytes_received_stop_download_callback(self): crawler = self.runner.create_crawler(BytesReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("failure")) @@ -505,7 +507,7 @@ class CrawlSpiderTestCase(TestCase): self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length) @defer.inlineCallbacks - def test_stop_download_errback(self): + def test_bytes_received_stop_download_errback(self): crawler = self.runner.create_crawler(BytesReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("response")) @@ -518,3 +520,23 @@ class CrawlSpiderTestCase(TestCase): self.assertLess( len(crawler.spider.meta["failure"].value.response.body), crawler.spider.full_response_length) + + @defer.inlineCallbacks + def test_headers_received_stop_download_callback(self): + crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider) + yield crawler.crawl(mockserver=self.mockserver) + self.assertIsNone(crawler.spider.meta.get("failure")) + self.assertIsInstance(crawler.spider.meta["response"], Response) + self.assertEqual(crawler.spider.meta["response"].headers, crawler.spider.meta.get("headers_received")) + + @defer.inlineCallbacks + def test_headers_received_stop_download_errback(self): + crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider) + yield crawler.crawl(mockserver=self.mockserver) + self.assertIsNone(crawler.spider.meta.get("response")) + self.assertIsInstance(crawler.spider.meta["failure"], Failure) + self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload) + self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response) + self.assertEqual( + crawler.spider.meta["failure"].value.response.headers, + crawler.spider.meta.get("headers_received")) diff --git a/tests/test_engine.py b/tests/test_engine.py index 3629aa1aa..ef1204f94 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -19,14 +19,12 @@ from urllib.parse import urlparse import attr from itemadapter import ItemAdapter from pydispatch import dispatcher -from testfixtures import LogCapture from twisted.internet import defer, reactor from twisted.trial import unittest from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor @@ -143,6 +141,7 @@ class CrawlerRun: self.reqreached = [] self.itemerror = [] self.itemresp = [] + self.headers = {} self.bytes = defaultdict(lambda: list()) self.signals_caught = {} self.spider_class = spider_class @@ -165,6 +164,7 @@ class CrawlerRun: self.crawler = get_crawler(self.spider_class) self.crawler.signals.connect(self.item_scraped, signals.item_scraped) self.crawler.signals.connect(self.item_error, signals.item_error) + self.crawler.signals.connect(self.headers_received, signals.headers_received) self.crawler.signals.connect(self.bytes_received, signals.bytes_received) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) @@ -183,6 +183,7 @@ class CrawlerRun: if not name.startswith('_'): disconnect_all(signal) self.deferred.callback(None) + return self.crawler.stop() def geturl(self, path): return f"http://localhost:{self.portno}{path}" @@ -197,6 +198,9 @@ class CrawlerRun: def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) + def headers_received(self, headers, body_length, request, spider): + self.headers[request] = headers + def bytes_received(self, data, request, spider): self.bytes[request].append(data) @@ -220,18 +224,7 @@ class CrawlerRun: self.signals_caught[sig] = signalargs -class StopDownloadCrawlerRun(CrawlerRun): - """ - Make sure raising the StopDownload exception stops the download of the response body - """ - - def bytes_received(self, data, request, spider): - super().bytes_received(data, request, spider) - raise StopDownload(fail=False) - - class EngineTest(unittest.TestCase): - @defer.inlineCallbacks def test_crawler(self): @@ -241,8 +234,8 @@ class EngineTest(unittest.TestCase): self.run = CrawlerRun(spider) yield self.run.run() self._assert_visited_urls() - self._assert_scheduled_requests(urls_to_visit=9) - self._assert_downloaded_responses() + self._assert_scheduled_requests(count=9) + self._assert_downloaded_responses(count=9) self._assert_scraped_items() self._assert_signals_caught() self._assert_bytes_received() @@ -251,7 +244,7 @@ class EngineTest(unittest.TestCase): def test_crawler_dupefilter(self): self.run = CrawlerRun(TestDupeFilterSpider) yield self.run.run() - self._assert_scheduled_requests(urls_to_visit=8) + self._assert_scheduled_requests(count=8) self._assert_dropped_requests() @defer.inlineCallbacks @@ -267,8 +260,8 @@ class EngineTest(unittest.TestCase): urls_expected = {self.run.geturl(p) for p in must_be_visited} assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" - def _assert_scheduled_requests(self, urls_to_visit=None): - self.assertEqual(urls_to_visit, len(self.run.reqplug)) + def _assert_scheduled_requests(self, count=None): + self.assertEqual(count, len(self.run.reqplug)) paths_expected = ['/item999.html', '/item2.html', '/item1.html'] @@ -286,10 +279,10 @@ class EngineTest(unittest.TestCase): def _assert_dropped_requests(self): self.assertEqual(len(self.run.reqdropped), 1) - def _assert_downloaded_responses(self): + def _assert_downloaded_responses(self, count): # response tests - self.assertEqual(9, len(self.run.respplug)) - self.assertEqual(9, len(self.run.reqreached)) + self.assertEqual(count, len(self.run.respplug)) + self.assertEqual(count, len(self.run.reqreached)) for response, _ in self.run.respplug: if self.run.getpath(response.url) == '/item999.html': @@ -323,6 +316,13 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) + def _assert_headers_received(self): + for headers in self.run.headers.values(): + self.assertIn(b"Server", headers) + self.assertIn(b"TwistedWeb", headers[b"Server"]) + self.assertIn(b"Date", headers) + self.assertIn(b"Content-Type", headers) + def _assert_bytes_received(self): self.assertEqual(9, len(self.run.bytes)) for request, data in self.run.bytes.items(): @@ -371,6 +371,7 @@ class EngineTest(unittest.TestCase): assert signals.spider_opened in self.run.signals_caught assert signals.spider_idle in self.run.signals_caught assert signals.spider_closed in self.run.signals_caught + assert signals.headers_received in self.run.signals_caught self.assertEqual({'spider': self.run.spider}, self.run.signals_caught[signals.spider_opened]) @@ -403,48 +404,6 @@ class EngineTest(unittest.TestCase): self.assertEqual(len(e.open_spiders), 0) -class StopDownloadEngineTest(EngineTest): - - @defer.inlineCallbacks - def test_crawler(self): - for spider in TestSpider, DictItemsSpider: - self.run = StopDownloadCrawlerRun(spider) - with LogCapture() as log: - yield self.run.run() - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler" - " StopDownloadCrawlerRun.bytes_received")) - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler" - " StopDownloadCrawlerRun.bytes_received")) - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler" - " StopDownloadCrawlerRun.bytes_received")) - self._assert_visited_urls() - self._assert_scheduled_requests(urls_to_visit=9) - self._assert_downloaded_responses() - self._assert_signals_caught() - self._assert_bytes_received() - - def _assert_bytes_received(self): - self.assertEqual(9, len(self.run.bytes)) - for request, data in self.run.bytes.items(): - joined_data = b"".join(data) - self.assertTrue(len(data) == 1) # signal was fired only once - if self.run.getpath(request.url) == "/numbers": - # Received bytes are not the complete response. The exact amount depends - # on the buffer size, which can vary, so we only check that the amount - # of received bytes is strictly less than the full response. - numbers = [str(x).encode("utf8") for x in range(2**18)] - self.assertTrue(len(joined_data) < len(b"".join(numbers))) - - if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == 'runserver': start_test_site(debug=True) diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py new file mode 100644 index 000000000..0ba69e096 --- /dev/null +++ b/tests/test_engine_stop_download_bytes.py @@ -0,0 +1,60 @@ +from testfixtures import LogCapture +from twisted.internet import defer + +from scrapy.exceptions import StopDownload + +from tests.test_engine import ( + AttrsItemsSpider, + DataClassItemsSpider, + DictItemsSpider, + TestSpider, + CrawlerRun, + EngineTest, +) + + +class BytesReceivedCrawlerRun(CrawlerRun): + def bytes_received(self, data, request, spider): + super().bytes_received(data, request, spider) + raise StopDownload(fail=False) + + +class BytesReceivedEngineTest(EngineTest): + @defer.inlineCallbacks + def test_crawler(self): + for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): + if spider is None: + continue + self.run = BytesReceivedCrawlerRun(spider) + with LogCapture() as log: + yield self.run.run() + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received")) + self._assert_visited_urls() + self._assert_scheduled_requests(count=9) + self._assert_downloaded_responses(count=9) + self._assert_signals_caught() + self._assert_headers_received() + self._assert_bytes_received() + + def _assert_bytes_received(self): + self.assertEqual(9, len(self.run.bytes)) + for request, data in self.run.bytes.items(): + joined_data = b"".join(data) + self.assertTrue(len(data) == 1) # signal was fired only once + if self.run.getpath(request.url) == "/numbers": + # Received bytes are not the complete response. The exact amount depends + # on the buffer size, which can vary, so we only check that the amount + # of received bytes is strictly less than the full response. + numbers = [str(x).encode("utf8") for x in range(2**18)] + self.assertTrue(len(joined_data) < len(b"".join(numbers))) diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py new file mode 100644 index 000000000..fad6643ad --- /dev/null +++ b/tests/test_engine_stop_download_headers.py @@ -0,0 +1,56 @@ +from testfixtures import LogCapture +from twisted.internet import defer + +from scrapy.exceptions import StopDownload + +from tests.test_engine import ( + AttrsItemsSpider, + DataClassItemsSpider, + DictItemsSpider, + TestSpider, + CrawlerRun, + EngineTest, +) + + +class HeadersReceivedCrawlerRun(CrawlerRun): + def headers_received(self, headers, body_length, request, spider): + super().headers_received(headers, body_length, request, spider) + raise StopDownload(fail=False) + + +class HeadersReceivedEngineTest(EngineTest): + @defer.inlineCallbacks + def test_crawler(self): + for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): + if spider is None: + continue + self.run = HeadersReceivedCrawlerRun(spider) + with LogCapture() as log: + yield self.run.run() + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from" + " signal handler HeadersReceivedCrawlerRun.headers_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from signal" + " handler HeadersReceivedCrawlerRun.headers_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from" + " signal handler HeadersReceivedCrawlerRun.headers_received")) + self._assert_visited_urls() + self._assert_downloaded_responses(count=6) + self._assert_signals_caught() + self._assert_bytes_received() + self._assert_headers_received() + + def _assert_bytes_received(self): + self.assertEqual(0, len(self.run.bytes)) + + def _assert_visited_urls(self): + must_be_visited = ["/", "/redirect", "/redirected"] + urls_visited = {rp[0].url for rp in self.run.respplug} + urls_expected = {self.run.geturl(p) for p in must_be_visited} + assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" From 6e5ea7924c7e3f5dd958d13db73e04c6348c99ba Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Fri, 12 Mar 2021 11:08:41 +0600 Subject: [PATCH 100/114] Log skipped urls by length to INFO, add skipped stats --- scrapy/spidermiddlewares/urllength.py | 17 ++++++---- tests/test_spidermiddleware_urllength.py | 43 +++++++++++++++++++----- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 5be1f80cb..ee3cb9fd6 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -14,22 +14,27 @@ logger = logging.getLogger(__name__) class UrlLengthMiddleware: - def __init__(self, maxlength): + def __init__(self, maxlength, stats): self.maxlength = maxlength + self.stats = stats @classmethod - def from_settings(cls, settings): + def from_crawler(cls, crawler): + settings = crawler.settings maxlength = settings.getint('URLLENGTH_LIMIT') if not maxlength: raise NotConfigured - return cls(maxlength) + return cls(maxlength, crawler.stats) def process_spider_output(self, response, result, spider): def _filter(request): if isinstance(request, Request) and len(request.url) > self.maxlength: - logger.debug("Ignoring link (url length > %(maxlength)d): %(url)s ", - {'maxlength': self.maxlength, 'url': request.url}, - extra={'spider': spider}) + logger.info( + "Ignoring link (url length > %(maxlength)d): %(url)s ", + {'maxlength': self.maxlength, 'url': request.url}, + extra={'spider': spider} + ) + self.stats.inc_value('urllength/request_ignored_count', spider=spider) return False else: return True diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 5ef2b23fd..33c524627 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,20 +1,45 @@ from unittest import TestCase +from testfixtures import LogCapture + from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spiders import Spider +from scrapy.statscollectors import StatsCollector +from scrapy.utils.test import get_crawler class TestUrlLengthMiddleware(TestCase): - def test_process_spider_output(self): - res = Response('http://scrapytest.org') + def setUp(self): + crawler = get_crawler(Spider) + self.spider = crawler._create_spider('foo') - short_url_req = Request('http://scrapytest.org/') - long_url_req = Request('http://scrapytest.org/this_is_a_long_url') - reqs = [short_url_req, long_url_req] + self.stats = StatsCollector(crawler) + self.stats.open_spider(self.spider) - mw = UrlLengthMiddleware(maxlength=25) - spider = Spider('foo') - out = list(mw.process_spider_output(res, reqs, spider)) - self.assertEqual(out, [short_url_req]) + self.maxlength = 25 + self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats) + + self.response = Response('http://scrapytest.org') + self.short_url_req = Request('http://scrapytest.org/') + self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url') + self.reqs = [self.short_url_req, self.long_url_req] + + def tearDown(self): + self.stats.close_spider(self.spider, '') + + def process_spider_output(self): + return list(self.mw.process_spider_output(self.response, self.reqs, self.spider)) + + def test_middleware_works(self): + self.assertEqual(self.process_spider_output(), [self.short_url_req]) + + def test_logging(self): + with LogCapture() as log: + self.process_spider_output() + + ric = self.stats.get_value('urllength/request_ignored_count', spider=self.spider) + self.assertEqual(ric, 1) + + self.assertIn(f'Ignoring link (url length > {self.maxlength})', str(log)) From d4b2b612551918647148013893da4cfa83fa2e7a Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Fri, 12 Mar 2021 16:59:37 +0600 Subject: [PATCH 101/114] Use from_settings for backward compatibility --- scrapy/spidermiddlewares/urllength.py | 10 ++++------ tests/test_spidermiddleware_urllength.py | 10 ++-------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index ee3cb9fd6..450d4ff40 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -14,17 +14,15 @@ logger = logging.getLogger(__name__) class UrlLengthMiddleware: - def __init__(self, maxlength, stats): + def __init__(self, maxlength): self.maxlength = maxlength - self.stats = stats @classmethod - def from_crawler(cls, crawler): - settings = crawler.settings + def from_settings(cls, settings): maxlength = settings.getint('URLLENGTH_LIMIT') if not maxlength: raise NotConfigured - return cls(maxlength, crawler.stats) + return cls(maxlength) def process_spider_output(self, response, result, spider): def _filter(request): @@ -34,7 +32,7 @@ class UrlLengthMiddleware: {'maxlength': self.maxlength, 'url': request.url}, extra={'spider': spider} ) - self.stats.inc_value('urllength/request_ignored_count', spider=spider) + spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider) return False else: return True diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 33c524627..6a72d2a8d 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -5,7 +5,6 @@ from testfixtures import LogCapture from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler @@ -14,21 +13,16 @@ class TestUrlLengthMiddleware(TestCase): def setUp(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider('foo') - - self.stats = StatsCollector(crawler) - self.stats.open_spider(self.spider) + self.stats = self.spider.crawler.stats self.maxlength = 25 - self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats) + self.mw = UrlLengthMiddleware(maxlength=self.maxlength) self.response = Response('http://scrapytest.org') self.short_url_req = Request('http://scrapytest.org/') self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url') self.reqs = [self.short_url_req, self.long_url_req] - def tearDown(self): - self.stats.close_spider(self.spider, '') - def process_spider_output(self): return list(self.mw.process_spider_output(self.response, self.reqs, self.spider)) From 0f254a6afbc3ad4a42048ea67acafdd035ba690a Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Fri, 12 Mar 2021 17:11:50 +0600 Subject: [PATCH 102/114] Test from_settings --- tests/test_spidermiddleware_urllength.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 6a72d2a8d..ee79c109f 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -6,17 +6,19 @@ from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +from scrapy.settings import Settings class TestUrlLengthMiddleware(TestCase): def setUp(self): + self.maxlength = 25 + settings = Settings({'URLLENGTH_LIMIT': self.maxlength}) + crawler = get_crawler(Spider) self.spider = crawler._create_spider('foo') self.stats = self.spider.crawler.stats - - self.maxlength = 25 - self.mw = UrlLengthMiddleware(maxlength=self.maxlength) + self.mw = UrlLengthMiddleware.from_settings(settings) self.response = Response('http://scrapytest.org') self.short_url_req = Request('http://scrapytest.org/') From 9cc4513bd60dcebcbfc53035482eff82d2b7acc0 Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Mon, 15 Mar 2021 21:38:03 +0600 Subject: [PATCH 103/114] simpler stats access --- tests/test_spidermiddleware_urllength.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index ee79c109f..171f4ddfd 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -17,7 +17,7 @@ class TestUrlLengthMiddleware(TestCase): crawler = get_crawler(Spider) self.spider = crawler._create_spider('foo') - self.stats = self.spider.crawler.stats + self.stats = crawler.stats self.mw = UrlLengthMiddleware.from_settings(settings) self.response = Response('http://scrapytest.org') From 2f61d7cc034e6ba6ba6ae9ccfc3cf2f1021b857e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 15 Mar 2021 14:25:46 -0300 Subject: [PATCH 104/114] Remove unnecesary del statement --- scrapy/core/http2/agent.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index a142fa210..f7b0c3f99 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -70,8 +70,6 @@ class H2ConnectionPool: d = pending_requests.popleft() d.callback(conn) - del pending_requests - return conn def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None: From 42e4dbb23de238e1252eb50b059666494418588f Mon Sep 17 00:00:00 2001 From: vinayak Date: Thu, 18 Mar 2021 18:10:03 +0530 Subject: [PATCH 105/114] Support Python 3.9 (#4759) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update .travis.yml * Update .travis.yml * updage travis.yml * Make 3.9 support official * Upgrade mitmproxy for Python 3.9 * Restore the Pylint job * Undo unintended change to mitmproxy requirement * Enable Python 3.9 in GitHub Actions * Work around reppy’s Python version limitation * Disable tests in Windows / Python 3.9 due to a Twisted bug Co-authored-by: Adrián Chaves --- .github/workflows/checks.yml | 10 ++++++---- .github/workflows/publish.yml | 4 ++-- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 7 ++++++- .github/workflows/tests-windows.yml | 4 ++++ .readthedocs.yml | 6 +++++- setup.py | 1 + tox.ini | 4 +++- 8 files changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2748bf5fe..02c647da9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -7,19 +7,21 @@ jobs: strategy: matrix: include: - - python-version: 3.8 + - python-version: 3.9 env: TOXENV: security - - python-version: 3.8 + - python-version: 3.9 env: TOXENV: flake8 + # Pylint requires installing reppy, which does not support Python 3.9 + # https://github.com/seomoz/reppy/issues/122 - python-version: 3.8 env: TOXENV: pylint - - python-version: 3.8 + - python-version: 3.9 env: TOXENV: typing - - python-version: 3.7 # Keep in sync with .readthedocs.yml + - python-version: 3.8 # Keep in sync with .readthedocs.yml env: TOXENV: docs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index aec6b8696..b48066ea4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,10 +9,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Check Tag id: check-release-tag diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 51d27c405..4f8f7a19d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -6,7 +6,7 @@ jobs: runs-on: macos-10.15 strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 89c0334e2..df5ee9d69 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -13,6 +13,9 @@ jobs: - python-version: 3.8 env: TOXENV: py + - python-version: 3.9 + env: + TOXENV: py - python-version: pypy3 env: TOXENV: pypy3 @@ -31,10 +34,12 @@ jobs: PYPY_VERSION: 3.6-v7.2.0 # extras + # extra-deps includes reppy, which does not support Python 3.9 + # https://github.com/seomoz/reppy/issues/122 - python-version: 3.8 env: TOXENV: extra-deps - - python-version: 3.8 + - python-version: 3.9 env: TOXENV: asyncio diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index ed2e4075d..5459a845b 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -16,6 +16,10 @@ jobs: - python-version: 3.8 env: TOXENV: py + # https://twistedmatrix.com/trac/ticket/9990 + #- python-version: 3.9 + #env: + #TOXENV: py steps: - uses: actions/checkout@v2 diff --git a/.readthedocs.yml b/.readthedocs.yml index e4d3f02cc..80a1cd036 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,10 +3,14 @@ formats: all sphinx: configuration: docs/conf.py fail_on_warning: true + +build: + image: latest + python: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image - version: 3.7 # Keep in sync with .travis.yml + version: 3.8 # Keep in sync with .github/workflows/checks.yml install: - requirements: docs/requirements.txt - path: . diff --git a/setup.py b/setup.py index b5c42a3c2..cf9261271 100644 --- a/setup.py +++ b/setup.py @@ -85,6 +85,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', diff --git a/tox.ini b/tox.ini index 69f52bd9f..9815f80f7 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,9 @@ deps = -rtests/requirements-py3.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 - mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy' + # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe + mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' + mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 From 8e73e1dfb51ad6a25ba4583f000d50a12718fb88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 18 Mar 2021 23:42:29 +0100 Subject: [PATCH 106/114] upper-constraints.txt: restrict botocore further --- tests/upper-constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/upper-constraints.txt b/tests/upper-constraints.txt index 75f337856..2a335e533 100644 --- a/tests/upper-constraints.txt +++ b/tests/upper-constraints.txt @@ -2,7 +2,7 @@ # pip dependency resolver from spending too much time backtracking. attrs>=20.2.0 Automat>=0.8.0 -botocore>=1.20.3 +botocore>=1.20.30 itemadapter>=0.1.1 itemloaders>=1.0.3 lxml>=4.6.1 From a390b934de4b8190499c15da115709aa6424d8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 18 Mar 2021 23:53:58 +0100 Subject: [PATCH 107/114] Do not install mitmproxy in Python 3.9 --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 86ae951b5..e0c69350e 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,8 @@ deps = # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe - mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' + # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 + #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras From 0dad0fce72266aa7b38b536f87bab26e7f233c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 19 Mar 2021 11:13:05 +0100 Subject: [PATCH 108/114] Use pip<20.3 to fix ReadTheDocs builds (#5052) --- .readthedocs.yml | 1 + docs/pip.txt | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 docs/pip.txt diff --git a/.readthedocs.yml b/.readthedocs.yml index 80a1cd036..2d781ae81 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,5 +12,6 @@ python: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image version: 3.8 # Keep in sync with .github/workflows/checks.yml install: + - requirements: docs/pip.txt - requirements: docs/requirements.txt - path: . diff --git a/docs/pip.txt b/docs/pip.txt new file mode 100644 index 000000000..095e53a0d --- /dev/null +++ b/docs/pip.txt @@ -0,0 +1,3 @@ +# In pip 20.3-21.0, the default dependency resolver causes the build in +# ReadTheDocs to fail due to memory exhaustion or timeout. +pip<20.3 From 308a58aa275aa514397351caa263e08de2f89adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 19 Mar 2021 18:39:44 +0100 Subject: [PATCH 109/114] Update CI to support Twisted 21.2.0 (#5027) --- scrapy/pipelines/images.py | 19 +++++++++++++------ tests/test_commands.py | 3 +++ tests/test_crawler.py | 4 ++++ tests/test_pipeline_crawl.py | 11 +++++++++++ tests/test_pipeline_images.py | 14 ++++++++++---- tests/test_pipeline_media.py | 11 +++++++++++ tox.ini | 4 +--- 7 files changed, 53 insertions(+), 13 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index aafd1d8b2..e3ab23ea5 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -9,9 +9,8 @@ from contextlib import suppress from io import BytesIO from itemadapter import ItemAdapter -from PIL import Image -from scrapy.exceptions import DropItem +from scrapy.exceptions import DropItem, NotConfigured from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline @@ -45,6 +44,14 @@ class ImagesPipeline(FilesPipeline): DEFAULT_IMAGES_RESULT_FIELD = 'images' def __init__(self, store_uri, download_func=None, settings=None): + try: + from PIL import Image + self._Image = Image + except ImportError: + raise NotConfigured( + 'ImagesPipeline requires installing Pillow 4.0.0 or later' + ) + super().__init__(store_uri, settings=settings, download_func=download_func) if isinstance(settings, dict) or settings is None: @@ -121,7 +128,7 @@ class ImagesPipeline(FilesPipeline): def get_images(self, response, request, info, *, item=None): path = self.file_path(request, response=response, info=info, item=item) - orig_image = Image.open(BytesIO(response.body)) + orig_image = self._Image.open(BytesIO(response.body)) width, height = orig_image.size if width < self.min_width or height < self.min_height: @@ -139,12 +146,12 @@ class ImagesPipeline(FilesPipeline): def convert_image(self, image, size=None): if image.format == 'PNG' and image.mode == 'RGBA': - background = Image.new('RGBA', image.size, (255, 255, 255)) + background = self._Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) image = background.convert('RGB') elif image.mode == 'P': image = image.convert("RGBA") - background = Image.new('RGBA', image.size, (255, 255, 255)) + background = self._Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) image = background.convert('RGB') elif image.mode != 'RGB': @@ -152,7 +159,7 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() - image.thumbnail(size, Image.ANTIALIAS) + image.thumbnail(size, self._Image.ANTIALIAS) buf = BytesIO() image.save(buf, 'JPEG') diff --git a/tests/test_commands.py b/tests/test_commands.py index d3ac05eac..eec1f02ee 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -17,6 +17,8 @@ from threading import Timer from unittest import skipIf from pytest import mark +from twisted import version as twisted_version +from twisted.python.versions import Version from twisted.trial import unittest import scrapy @@ -630,6 +632,7 @@ class MySpider(scrapy.Spider): @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') def test_custom_asyncio_loop_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ab113710d..dec517bb6 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -8,7 +8,9 @@ from unittest import skipIf from pytest import raises, mark from testfixtures import LogCapture +from twisted import version as twisted_version from twisted.internet import defer +from twisted.python.versions import Version from twisted.trial import unittest import scrapy @@ -358,6 +360,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") self.assertIn("Spider closed (finished)", log) @@ -366,6 +369,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly") @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows") + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 55fcfa7ba..f49fda701 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -180,7 +180,18 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) +try: + from PIL import Image # noqa: imported just to check for the import error +except ImportError: + skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' +else: + skip_pillow = None + + class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase): + + skip = skip_pillow + pipeline_class = 'scrapy.pipelines.images.ImagesPipeline' store_setting_key = 'IMAGES_STORE' media_key = 'images' diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index ad138a2dc..c69cd0e4a 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -23,15 +23,16 @@ except ImportError: dataclass_field = None -skip = False try: from PIL import Image except ImportError: - skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' + skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' else: encoders = {'jpeg_encoder', 'jpeg_decoder'} if not encoders.issubset(set(Image.core.__dict__)): - skip = 'Missing JPEG encoders' + skip_pillow = 'Missing JPEG encoders' + else: + skip_pillow = None def _mocked_download_func(request, info): @@ -41,7 +42,7 @@ def _mocked_download_func(request, info): class ImagesPipelineTestCase(unittest.TestCase): - skip = skip + skip = skip_pillow def setUp(self): self.tempdir = mkdtemp() @@ -137,6 +138,8 @@ class DeprecatedImagesPipeline(ImagesPipeline): class ImagesPipelineTestCaseFieldsMixin: + skip = skip_pillow + def test_item_fields_default(self): url = 'http://www.example.com/images/1.jpg' item = self.item_class(name='item1', image_urls=[url]) @@ -221,6 +224,9 @@ class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, u class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): + + skip = skip_pillow + img_cls_attribute_names = [ # Pipeline attribute names with corresponding setting names. ("EXPIRES", "IMAGES_EXPIRES"), diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 6afd47497..893d43052 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,3 +1,5 @@ +from typing import Optional + from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure @@ -17,6 +19,14 @@ from scrapy.utils.signal import disconnect_all from scrapy import signals +try: + from PIL import Image # noqa: imported just to check for the import error +except ImportError: + skip_pillow: Optional[str] = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' +else: + skip_pillow = None + + def _mocked_download_func(request, info): response = request.meta.get('response') return response() if callable(response) else response @@ -379,6 +389,7 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): + skip = skip_pillow def setUp(self): self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func) diff --git a/tox.ini b/tox.ini index e0c69350e..6907c8906 100644 --- a/tox.ini +++ b/tox.ini @@ -19,9 +19,6 @@ deps = mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 - Pillow>=4.0.0 - # Twisted 21+ causes issues in tests that use skipIf - Twisted[http2]>=17.9.0,<21 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -124,6 +121,7 @@ deps = {[testenv]deps} reppy robotexclusionrulesparser + Pillow>=4.0.0 [testenv:asyncio] commands = From 2973d8d51abbda4839981c192a366e907e2ad39e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Mar 2021 11:24:10 +0100 Subject: [PATCH 110/114] Remove unnecessary reference to private parsel.Selector._default_type (#5006) --- scrapy/selector/unified.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index a25871433..08f08e8d7 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -69,7 +69,7 @@ class Selector(_ParselSelector, object_ref): raise ValueError(f'{self.__class__.__name__}.__init__() received ' 'both response and text') - st = _st(response, type or self._default_type) + st = _st(response, type) if text is not None: response = _response_from_text(text, st) From ec5a7918ec2d8888ba356f9e3295dcd1ee935884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Mar 2021 11:25:40 +0100 Subject: [PATCH 111/114] Include Content-Length in HTTP/1.1 responses (#5057) --- scrapy/core/downloader/handlers/http11.py | 13 +++++++++++-- tests/test_downloader_handlers.py | 7 +++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9cdadb27f..1f82751fd 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -358,10 +358,19 @@ class ScrapyAgent: request.meta['download_latency'] = time() - start_time return result + @staticmethod + def _headers_from_twisted_response(response): + headers = Headers() + if response.length is not None: + headers[b'Content-Length'] = str(response.length).encode() + for key, value in response.headers.getAllRawHeaders(): + headers[key] = value + return headers + def _cb_bodyready(self, txresponse, request): headers_received_result = self._crawler.signals.send_catch_log( signal=signals.headers_received, - headers=Headers(txresponse.headers.getAllRawHeaders()), + headers=self._headers_from_twisted_response(txresponse), body_length=txresponse.length, request=request, spider=self._crawler.spider, @@ -435,7 +444,7 @@ class ScrapyAgent: return d def _cb_bodydone(self, result, request, url): - headers = Headers(result["txresponse"].headers.getAllRawHeaders()) + headers = self._headers_from_twisted_response(result["txresponse"]) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) try: version = result["txresponse"].version diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 86d72772c..fa7d5c8a6 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -364,6 +364,13 @@ class HttpTestCase(unittest.TestCase): d.addCallback(self.assertEqual, body) return d + def test_response_header_content_length(self): + request = Request(self.getURL("file"), method=b"GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.headers[b'content-length']) + d.addCallback(self.assertEqual, b'159') + return d + class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" From 72e8cea8afb85f6704fcf6f5648b0a01f1abaab3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 22 Mar 2021 11:51:11 -0300 Subject: [PATCH 112/114] Avoid exceptions in is_generator_with_return_value (#4935) --- scrapy/utils/misc.py | 27 ++- ...t_return_with_argument_inside_generator.py | 167 +++++++++++++++--- 2 files changed, 161 insertions(+), 33 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 081cd33f1..5c986eedc 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -9,7 +9,6 @@ from collections import deque from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules -from textwrap import dedent from w3lib.html import replace_entities @@ -227,7 +226,8 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - tree = ast.parse(dedent(inspect.getsource(callable))) + code = re.sub(r"^[\t ]+", "", inspect.getsource(callable)) + tree = ast.parse(code) for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): _generator_callbacks_cache[callable] = True @@ -242,12 +242,23 @@ def warn_on_generator_with_return_value(spider, callable): Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None """ - if is_generator_with_return_value(callable): + try: + if is_generator_with_return_value(callable): + warnings.warn( + f'The "{spider.__class__.__name__}.{callable.__name__}" method is ' + 'a generator and includes a "return" statement with a value ' + 'different than None. This could lead to unexpected behaviour. Please see ' + 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' + 'for details about the semantics of the "return" statement within generators', + stacklevel=2, + ) + except IndentationError: + callable_name = spider.__class__.__name__ + "." + callable.__name__ warnings.warn( - f'The "{spider.__class__.__name__}.{callable.__name__}" method is ' - 'a generator and includes a "return" statement with a value ' - 'different than None. This could lead to unexpected behaviour. Please see ' - 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' - 'for details about the semantics of the "return" statement within generators', + f'Unable to determine whether or not "{callable_name}" is a generator with a return value. ' + 'This will not prevent your code from working, but it prevents Scrapy from detecting ' + f'potential issues in your implementation of "{callable_name}". Please, report this in the ' + 'Scrapy issue tracker (https://github.com/scrapy/scrapy/issues), ' + f'including the code of "{callable_name}"', stacklevel=2, ) diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 2be38620c..1c85ca353 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,35 +1,116 @@ import unittest +import warnings +from unittest import mock -from scrapy.utils.misc import is_generator_with_return_value +from scrapy.utils.misc import is_generator_with_return_value, warn_on_generator_with_return_value + + +def _indentation_error(*args, **kwargs): + raise IndentationError() + + +def top_level_return_something(): + """ +docstring + """ + url = """ +https://example.org +""" + yield url + return 1 + + +def top_level_return_none(): + """ +docstring + """ + url = """ +https://example.org +""" + yield url + return + + +def generator_that_returns_stuff(): + yield 1 + yield 2 + return 3 class UtilsMiscPy3TestCase(unittest.TestCase): - def test_generators_with_return_statements(self): - def f(): + def test_generators_return_something(self): + def f1(): yield 1 return 2 - def g(): + def g1(): yield 1 - return 'asdf' + return "asdf" - def h(): + def h1(): + yield 1 + + def helper(): + return 0 + + yield helper() + return 2 + + def i1(): + """ +docstring + """ + url = """ +https://example.org + """ + yield url + return 1 + + assert is_generator_with_return_value(top_level_return_something) + assert is_generator_with_return_value(f1) + assert is_generator_with_return_value(g1) + assert is_generator_with_return_value(h1) + assert is_generator_with_return_value(i1) + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_something) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.top_level_return_something" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, f1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.f1" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, g1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.g1" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, h1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.h1" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, i1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.i1" method is a generator', str(w[0].message)) + + def test_generators_return_none(self): + def f2(): yield 1 return None - def i(): + def g2(): yield 1 return - def j(): + def h2(): yield 1 - def k(): + def i2(): yield 1 - yield from g() + yield from generator_that_returns_stuff() - def m(): + def j2(): yield 1 def helper(): @@ -37,20 +118,56 @@ class UtilsMiscPy3TestCase(unittest.TestCase): yield helper() - def n(): - yield 1 + def k2(): + """ +docstring + """ + url = """ +https://example.org + """ + yield url + return - def helper(): - return 0 + def l2(): + return - yield helper() - return 2 + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f2) + assert not is_generator_with_return_value(g2) + assert not is_generator_with_return_value(h2) + assert not is_generator_with_return_value(i2) + assert not is_generator_with_return_value(j2) # not recursive + assert not is_generator_with_return_value(k2) # not recursive + assert not is_generator_with_return_value(l2) - assert is_generator_with_return_value(f) - assert is_generator_with_return_value(g) - assert not is_generator_with_return_value(h) - assert not is_generator_with_return_value(i) - assert not is_generator_with_return_value(j) - assert not is_generator_with_return_value(k) # not recursive - assert not is_generator_with_return_value(m) - assert is_generator_with_return_value(n) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_none) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, f2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, g2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, h2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, i2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, j2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, k2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, l2) + self.assertEqual(len(w), 0) + + @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error) + def test_indentation_error(self): + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_none) + self.assertEqual(len(w), 1) + self.assertIn('Unable to determine', str(w[0].message)) From f0e1a33225dd7c9162e277626df70284f41a427e Mon Sep 17 00:00:00 2001 From: Pratik Mahankal <53421565+pratik1500@users.noreply.github.com> Date: Tue, 23 Mar 2021 22:46:50 +0530 Subject: [PATCH 113/114] Sort the list of Request.meta alphabetically #5061 (#5065) --- docs/topics/request-response.rst | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a0448c5ab..c0283df01 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -363,26 +363,26 @@ are some special keys recognized by Scrapy and its built-in extensions. Those are: -* :reqmeta:`dont_redirect` -* :reqmeta:`dont_retry` -* :reqmeta:`handle_httpstatus_list` -* :reqmeta:`handle_httpstatus_all` -* :reqmeta:`dont_merge_cookies` +* :reqmeta:`bindaddress` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` +* :reqmeta:`dont_merge_cookies` +* :reqmeta:`dont_obey_robotstxt` +* :reqmeta:`dont_redirect` +* :reqmeta:`dont_retry` +* :reqmeta:`download_fail_on_dataloss` +* :reqmeta:`download_latency` +* :reqmeta:`download_maxsize` +* :reqmeta:`download_timeout` +* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) +* ``ftp_user`` (See :setting:`FTP_USER` for more info) +* :reqmeta:`handle_httpstatus_all` +* :reqmeta:`handle_httpstatus_list` +* :reqmeta:`max_retry_times` +* :reqmeta:`proxy` * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` -* :reqmeta:`bindaddress` -* :reqmeta:`dont_obey_robotstxt` -* :reqmeta:`download_timeout` -* :reqmeta:`download_maxsize` -* :reqmeta:`download_latency` -* :reqmeta:`download_fail_on_dataloss` -* :reqmeta:`proxy` -* ``ftp_user`` (See :setting:`FTP_USER` for more info) -* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * :reqmeta:`referrer_policy` -* :reqmeta:`max_retry_times` .. reqmeta:: bindaddress From 9c9e1a318d83b8e55125fdf7a7fb9482cb4951f8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 25 Mar 2021 11:58:39 -0300 Subject: [PATCH 114/114] [HTTP/1.1] Skip Content-Length header if its value is UNKNOWN_LENGTH (#5062) --- scrapy/core/downloader/handlers/http11.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1f82751fd..25cb3ec62 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -361,10 +361,9 @@ class ScrapyAgent: @staticmethod def _headers_from_twisted_response(response): headers = Headers() - if response.length is not None: + if response.length != UNKNOWN_LENGTH: headers[b'Content-Length'] = str(response.length).encode() - for key, value in response.headers.getAllRawHeaders(): - headers[key] = value + headers.update(response.headers.getAllRawHeaders()) return headers def _cb_bodyready(self, txresponse, request):