From df937d8280fe0781f6cf1715a3a0cd28c6e94eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Feb 2020 22:33:36 +0100 Subject: [PATCH 1/5] Implement Response.cb_kwargs --- docs/topics/request-response.rst | 12 ++++++++++++ scrapy/http/response/__init__.py | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 4cf367d96..05b7bb5c7 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -672,6 +672,18 @@ Response objects .. seealso:: :attr:`Request.meta` attribute + .. attribute:: Response.cb_kwargs + + A shortcut to the :attr:`Request.cb_kwargs` attribute of the + :attr:`Response.request` object (ie. ``self.request.cb_kwargs``). + + Unlike the :attr:`Response.request` attribute, the + :attr:`Response.cb_kwargs` attribute is propagated along redirects and + retries, so you will get the original :attr:`Request.cb_kwargs` sent + from your spider. + + .. seealso:: :attr:`Request.cb_kwargs` attribute + .. attribute:: Response.flags A list that contains flags for this response. Flags are labels used for diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 64e9c6c20..ee9720d52 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -24,6 +24,16 @@ class Response(object_ref): self.request = request self.flags = [] if flags is None else list(flags) + @property + def cb_kwargs(self): + try: + return self.request.cb_kwargs + except AttributeError: + raise AttributeError( + "Response.cb_kwargs not available, this response " + "is not tied to any request" + ) + @property def meta(self): try: From 5ff9eb90ea9d533d3f960db75071f0fe638503ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Feb 2020 22:36:18 +0100 Subject: [PATCH 2/5] Add a test for the copy of cb_kwargs from Request to Response --- tests/test_http_response.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 960ecea3e..39f5fe750 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -72,6 +72,12 @@ class BaseResponseTest(unittest.TestCase): r1 = self.response_class("http://www.example.com", body=b"Some body", request=req) assert r1.meta is req.meta + def test_copy_cb_kwargs(self): + req = Request("http://www.example.com") + req.cb_kwargs['foo'] = 'bar' + r1 = self.response_class("http://www.example.com", body=b"Some body", request=req) + assert r1.cb_kwargs is req.cb_kwargs + def test_copy_inherited_classes(self): """Test Response children copies preserve their class""" From 43b43654a1dacae7b63fc067dc69929c262d9a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Feb 2020 22:39:58 +0100 Subject: [PATCH 3/5] Add tests for meta and cb_kwargs not being available --- tests/test_http_response.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 39f5fe750..5a19f9d54 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -78,6 +78,16 @@ class BaseResponseTest(unittest.TestCase): r1 = self.response_class("http://www.example.com", body=b"Some body", request=req) assert r1.cb_kwargs is req.cb_kwargs + def test_unavailable_meta(self): + r1 = self.response_class("http://www.example.com", body=b"Some body") + with self.assertRaisesRegex(AttributeError, r'Response\.meta not available'): + r1.meta + + def test_unavailable_cb_kwargs(self): + r1 = self.response_class("http://www.example.com", body=b"Some body") + with self.assertRaisesRegex(AttributeError, r'Response\.cb_kwargs not available'): + r1.cb_kwargs + def test_copy_inherited_classes(self): """Test Response children copies preserve their class""" From 5ae3e1678fa99b3c44cb8981079df51ec34b860f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 14 Feb 2020 22:30:36 +0100 Subject: [PATCH 4/5] =?UTF-8?q?ie.=20=E2=86=92=20i.e.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: elacuesta --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 05b7bb5c7..260fe3caf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -675,7 +675,7 @@ Response objects .. attribute:: Response.cb_kwargs A shortcut to the :attr:`Request.cb_kwargs` attribute of the - :attr:`Response.request` object (ie. ``self.request.cb_kwargs``). + :attr:`Response.request` object (i.e. ``self.request.cb_kwargs``). Unlike the :attr:`Response.request` attribute, the :attr:`Response.cb_kwargs` attribute is propagated along redirects and From a04dd13cd08f1ff392a8bbe284fa0c8fe8924b57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 14 Feb 2020 22:31:30 +0100 Subject: [PATCH 5/5] =?UTF-8?q?ie.=20=E2=86=92=20i.e.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/request-response.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 260fe3caf..d6c7cbec9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -664,7 +664,7 @@ Response objects .. attribute:: Response.meta A shortcut to the :attr:`Request.meta` attribute of the - :attr:`Response.request` object (ie. ``self.request.meta``). + :attr:`Response.request` object (i.e. ``self.request.meta``). Unlike the :attr:`Response.request` attribute, the :attr:`Response.meta` attribute is propagated along redirects and retries, so you will get @@ -770,7 +770,7 @@ TextResponse objects 1. the encoding passed in the ``__init__`` method ``encoding`` argument 2. the encoding declared in the Content-Type HTTP header. If this - encoding is not valid (ie. unknown), it is ignored and the next + encoding is not valid (i.e. unknown), it is ignored and the next resolution mechanism is tried. 3. the encoding declared in the response body. The TextResponse class