From 99cc853d6953d336ca65e0eecc0cb3286306bacf Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 29 Sep 2020 23:53:37 -0300 Subject: [PATCH 1/9] Response.protocol attribute --- scrapy/core/downloader/handlers/http11.py | 1 + scrapy/http/response/__init__.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1b041c8a8..0f30b01f9 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -442,6 +442,7 @@ class ScrapyAgent: flags=result["flags"], certificate=result["certificate"], ip_address=result["ip_address"], + protocol=getattr(result["txresponse"], "version", None), ) if result.get("failure"): result["failure"].value.response = response diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index c635fde69..f4ef79c72 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -17,8 +17,18 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): - def __init__(self, url, status=200, headers=None, body=b'', flags=None, - request=None, certificate=None, ip_address=None): + def __init__( + self, + url, + status=200, + headers=None, + body=b"", + flags=None, + request=None, + certificate=None, + ip_address=None, + protocol=None, + ): self.headers = Headers(headers or {}) self.status = int(status) self._set_body(body) @@ -27,6 +37,7 @@ class Response(object_ref): self.flags = [] if flags is None else list(flags) self.certificate = certificate self.ip_address = ip_address + self.protocol = protocol @property def cb_kwargs(self): @@ -90,7 +101,7 @@ class Response(object_ref): given new values. """ for x in ['url', 'status', 'headers', 'body', - 'request', 'flags', 'certificate', 'ip_address']: + 'request', 'flags', 'certificate', 'ip_address', 'protocol']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) From 5b6b56240c24d02ef69e6cc591ffb2529bc3f36a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:08:37 -0300 Subject: [PATCH 2/9] Test Response.protocol attribute --- tests/test_downloader_handlers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 3e8d7e6b9..eb6d40df7 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -489,6 +489,13 @@ class Http11TestCase(HttpTestCase): def test_download_broken_chunked_content_allow_data_loss_via_setting(self): return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked') + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, (b"HTTP", 1, 1)) + return d + class Https11TestCase(Http11TestCase): scheme = 'https' From 587b4dd71fca12fa5fcc766b891540af77cb27c8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:20:50 -0300 Subject: [PATCH 3/9] Docs for the Response.protocol attribute --- docs/topics/request-response.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f3aaa2c8f..d7d5cd44e 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -693,9 +693,22 @@ Response objects :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` + :param protocol: A tuple containing information about the protocol that was used + to download the response. Taken from the ``version`` attribute of the + corresponding :class:`twisted.web.client.Response` object, it will tipically + consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` + to represent "HTTP/1.1". + :type protocol: :class:`tuple` + + .. versionadded:: 2.X.X + The ``protocol`` parameter. + .. versionadded:: 2.1.0 The ``ip_address`` parameter. + .. versionadded:: 2.0.0 + The ``certificate`` parameter. + .. attribute:: Response.url A string containing the URL of the response. @@ -780,6 +793,8 @@ Response objects .. attribute:: Response.certificate + .. versionadded:: 2.0.0 + A :class:`twisted.internet.ssl.Certificate` object representing the server's SSL certificate. @@ -795,6 +810,20 @@ Response objects handler, i.e. for ``http(s)`` responses. For other handlers, :attr:`ip_address` is always ``None``. + .. attribute:: Response.protocol + + .. versionadded:: 2.X.X + + A tuple containing information about the protocol that was used + to download the response. Taken from the ``version`` attribute of the + corresponding :class:`twisted.web.client.Response` object, it will tipically + consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` + to represent "HTTP/1.1". + + This attribute is currently only populated by the HTTP 1.1 download + handler, i.e. for ``http(s)`` responses. For other handlers, + :attr:`protocol` is always ``None``. + .. method:: Response.copy() Returns a new Response which is a copy of this Response. From 0fb7bcb2cf1606f63f1863bea254a34386c6a0f1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:26:03 -0300 Subject: [PATCH 4/9] Style adjustment --- scrapy/http/response/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f4ef79c72..185a9bb67 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -100,8 +100,9 @@ class Response(object_ref): """Create a new Response with the same attributes except for those given new values. """ - for x in ['url', 'status', 'headers', 'body', - 'request', 'flags', 'certificate', 'ip_address', 'protocol']: + for x in [ + "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol", + ]: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) From 61d089485c7ba66649b936e34833b2013fa12458 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:31:15 -0300 Subject: [PATCH 5/9] Docs: sort versionadded directives --- docs/topics/request-response.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d7d5cd44e..f1be41dde 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -700,14 +700,14 @@ Response objects to represent "HTTP/1.1". :type protocol: :class:`tuple` - .. versionadded:: 2.X.X - The ``protocol`` parameter. + .. versionadded:: 2.0.0 + The ``certificate`` parameter. .. versionadded:: 2.1.0 The ``ip_address`` parameter. - .. versionadded:: 2.0.0 - The ``certificate`` parameter. + .. versionadded:: 2.X.X + The ``protocol`` parameter. .. attribute:: Response.url From 22424125560496c9d131c9b7226aaf0f794e5ad8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 10:50:54 -0300 Subject: [PATCH 6/9] Docs: placeholder for versionadded directive --- docs/topics/request-response.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f1be41dde..1cb724227 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -706,7 +706,7 @@ Response objects .. versionadded:: 2.1.0 The ``ip_address`` parameter. - .. versionadded:: 2.X.X + .. versionadded:: VERSION The ``protocol`` parameter. .. attribute:: Response.url @@ -812,7 +812,7 @@ Response objects .. attribute:: Response.protocol - .. versionadded:: 2.X.X + .. versionadded:: VERSION A tuple containing information about the protocol that was used to download the response. Taken from the ``version`` attribute of the From 5e9a99e6a1e940864d9157252592f04658eaf851 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 11:15:29 -0300 Subject: [PATCH 7/9] Reponse.protocol as string --- docs/topics/request-response.rst | 20 +++++++------------- scrapy/core/downloader/handlers/http11.py | 7 ++++++- scrapy/core/downloader/webclient.py | 4 ++-- tests/test_downloader_handlers.py | 9 ++++++++- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1cb724227..98906992d 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -693,12 +693,9 @@ Response objects :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` - :param protocol: A tuple containing information about the protocol that was used - to download the response. Taken from the ``version`` attribute of the - corresponding :class:`twisted.web.client.Response` object, it will tipically - consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` - to represent "HTTP/1.1". - :type protocol: :class:`tuple` + :param protocol: The protocol that was used to download the response. + For instance: "HTTP/1.0", "HTTP/1.1" + :type protocol: :class:`str` .. versionadded:: 2.0.0 The ``certificate`` parameter. @@ -814,14 +811,11 @@ Response objects .. versionadded:: VERSION - A tuple containing information about the protocol that was used - to download the response. Taken from the ``version`` attribute of the - corresponding :class:`twisted.web.client.Response` object, it will tipically - consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` - to represent "HTTP/1.1". + The protocol that was used to download the response. + For instance: "HTTP/1.0", "HTTP/1.1" - This attribute is currently only populated by the HTTP 1.1 download - handler, i.e. for ``http(s)`` responses. For other handlers, + This attribute is currently only populated by the HTTP download + handlers, i.e. for ``http(s)`` responses. For other handlers, :attr:`protocol` is always ``None``. .. method:: Response.copy() diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 0f30b01f9..c7553eb87 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -434,6 +434,11 @@ class ScrapyAgent: def _cb_bodydone(self, result, request, url): headers = Headers(result["txresponse"].headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) + try: + version = result["txresponse"].version + protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}" + except (AttributeError, TypeError): + protocol = None response = respcls( url=url, status=int(result["txresponse"].code), @@ -442,7 +447,7 @@ class ScrapyAgent: flags=result["flags"], certificate=result["certificate"], ip_address=result["ip_address"], - protocol=getattr(result["txresponse"], "version", None), + protocol=protocol, ) if result.get("failure"): result["failure"].value.response = response diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index c13683393..9524cce2b 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -7,7 +7,7 @@ from twisted.internet.protocol import ClientFactory from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, to_unicode from scrapy.responsetypes import responsetypes @@ -110,7 +110,7 @@ class ScrapyHTTPClientFactory(ClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url) - return respcls(url=self._url, status=status, headers=headers, body=body) + return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version)) def _set_connection_attributes(self, request): parsed = urlparse_cached(request) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index eb6d40df7..a8763a7a5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -360,6 +360,13 @@ class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" download_handler_cls = HTTP10DownloadHandler + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.0") + return d + class Https10TestCase(Http10TestCase): scheme = 'https' @@ -493,7 +500,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL("host"), method="GET") d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, (b"HTTP", 1, 1)) + d.addCallback(self.assertEqual, "HTTP/1.1") return d From b0368228d7f6391c0df41fca1609c6548613ad6b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 11:18:03 -0300 Subject: [PATCH 8/9] Add exception to catch --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c7553eb87..a0fd837b1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -437,7 +437,7 @@ class ScrapyAgent: try: version = result["txresponse"].version protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}" - except (AttributeError, TypeError): + except (AttributeError, TypeError, IndexError): protocol = None response = respcls( url=url, From 2405df49f14cbc052d73e58a819a87417b2502e8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 16 Nov 2020 12:50:33 -0300 Subject: [PATCH 9/9] Add tests for Response.protocol=None --- tests/test_downloader_handlers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a8763a7a5..f51a6cd3c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -115,6 +115,7 @@ class FileTestCase(unittest.TestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.status, 200) self.assertEqual(response.body, b'0123456789') + self.assertEqual(response.protocol, None) request = Request(path_to_file_uri(self.tmpname + '^')) assert request.url.upper().endswith('%5E') @@ -976,6 +977,7 @@ class BaseFTPTestCase(unittest.TestCase): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'I have the power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) + self.assertIsNone(r.protocol) return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): @@ -1134,3 +1136,10 @@ class DataURITestCase(unittest.TestCase): request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') return self.download_request(request, self.spider).addCallback(_test) + + def test_protocol(self): + def _test(response): + self.assertIsNone(response.protocol) + + request = Request("data:,") + return self.download_request(request, self.spider).addCallback(_test)