From 0f0690b1b65acd0087156f105efaea89b7be82eb Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 10 Nov 2023 12:16:16 +0200 Subject: [PATCH 1/5] cookies_tests added new Echo2 endpoint --- tests/mockserver.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/mockserver.py b/tests/mockserver.py index 647b0682e..cea79530c 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -200,6 +200,18 @@ class Echo(LeafResource): render_POST = render_GET +class Echo2(LeafResource): + def render_GET(self, request): + req = json.loads(request.content.read()) + + request.setResponseCode(req.get("status", 200)) + for k, v in req.get("headers").items(): + request.setHeader(bytes(k, encoding="utf8"), bytes(v, encoding="utf8")) + return bytes(req.get("body", ""), encoding="utf8") + + render_POST = render_GET + + class RedirectTo(LeafResource): def render(self, request): goto = getarg(request, b"goto", b"/") @@ -249,6 +261,7 @@ class Root(resource.Resource): self.putChild(b"drop", Drop()) self.putChild(b"raw", Raw()) self.putChild(b"echo", Echo()) + self.putChild(b"echo2", Echo2()) self.putChild(b"payload", PayloadResource()) self.putChild( b"xpayload", From 77dc96d4de5f4d065d69af23ae41f069bd98b774 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 10 Nov 2023 12:24:38 +0200 Subject: [PATCH 2/5] cookies_tests new test added --- tests/test_cookies.py | 68 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 tests/test_cookies.py diff --git a/tests/test_cookies.py b/tests/test_cookies.py new file mode 100644 index 000000000..9b327d8c1 --- /dev/null +++ b/tests/test_cookies.py @@ -0,0 +1,68 @@ +import json + +from twisted.internet import defer +from twisted.trial.unittest import TestCase + +from scrapy.crawler import CrawlerRunner +from scrapy.http import Request +from tests.mockserver import MockServer +from tests.spiders import MetaSpider + + +class CookiesSpider(MetaSpider): + name = "cookies" + + def start_requests(self): + url = self.mockserver.url("/echo2/") + for status in [200, 302, 503]: + # Response params that mockserver will return + body = { + "status": status, + "headers": { + "Set-cookie": f"status={status}", + }, + "body": "", + } + + if status == 302: + body["headers"]["location"] = self.mockserver.url("/") + + yield Request( + url, + callback=self.parse, + meta={"cookiejar": status}, + body=json.dumps(body), + ) + + def parse(self, response): + pass + + def close(self, spider): + mw = [ + m + for m in self.crawler.engine.downloader.middleware.middlewares + if "Cookies" in str(type(m)) + ][0] + jars_data = {} + for k, jar in mw.jars.items(): + jars_data[k] = { + s: v.value for s, v in jar._cookies["127.0.0.1"]["/echo2"].items() + } + self.crawler.stats._stats["cookies_values"] = jars_data + + +class CrawlTestCase(TestCase): + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self.runner = CrawlerRunner() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_delay(self): + crawler = CrawlerRunner().create_crawler(CookiesSpider) + yield crawler.crawl(mockserver=self.mockserver) + cookies_stats = crawler.stats._stats.get("cookies_values") + self.assertTrue(cookies_stats) # WIP From 6d875221d3f3f531bc3fec74d178770d195f0b90 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 19 Jan 2024 16:30:40 +0200 Subject: [PATCH 3/5] cookies_tests test updated --- tests/test_cookies.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_cookies.py b/tests/test_cookies.py index 9b327d8c1..a79bfdf55 100644 --- a/tests/test_cookies.py +++ b/tests/test_cookies.py @@ -65,4 +65,9 @@ class CrawlTestCase(TestCase): crawler = CrawlerRunner().create_crawler(CookiesSpider) yield crawler.crawl(mockserver=self.mockserver) cookies_stats = crawler.stats._stats.get("cookies_values") - self.assertTrue(cookies_stats) # WIP + expected_cookies = { + 200: {"status": "200"}, + 302: {"status": "302"}, + 503: {"status": "503"}, + } + self.assertEqual(cookies_stats, expected_cookies) From 1e14be77434ce2c71e0559d99bf0ecca91827a5e Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 19 Jan 2024 16:31:24 +0200 Subject: [PATCH 4/5] cookies_tests naming updated --- tests/test_cookies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cookies.py b/tests/test_cookies.py index a79bfdf55..db78f9f18 100644 --- a/tests/test_cookies.py +++ b/tests/test_cookies.py @@ -51,7 +51,7 @@ class CookiesSpider(MetaSpider): self.crawler.stats._stats["cookies_values"] = jars_data -class CrawlTestCase(TestCase): +class CookiesTestCase(TestCase): def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() @@ -61,7 +61,7 @@ class CrawlTestCase(TestCase): self.mockserver.__exit__(None, None, None) @defer.inlineCallbacks - def test_delay(self): + def test_cookies(self): crawler = CrawlerRunner().create_crawler(CookiesSpider) yield crawler.crawl(mockserver=self.mockserver) cookies_stats = crawler.stats._stats.get("cookies_values") From 689e7628965cb6b80779eb09f99ab9dd9ae3f9f2 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 6 Aug 2026 15:35:12 +0200 Subject: [PATCH 5/5] Allow setting the response status code in the mockserver response-headers resource --- tests/mockserver/http_resources.py | 4 +- tests/test_cookies.py | 73 --------------------- tests/utils/bases/download_handlers_http.py | 14 ++++ 3 files changed, 17 insertions(+), 74 deletions(-) delete mode 100644 tests/test_cookies.py diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py index cb028bc10..d86a3d6df 100644 --- a/tests/mockserver/http_resources.py +++ b/tests/mockserver/http_resources.py @@ -380,11 +380,13 @@ class UriResource(BaseResource): class ResponseHeadersResource(BaseResource): - """Return a response with headers set from the JSON request body""" + """Return a response with headers set from the JSON request body, and the + status code set from the *n* URL parameter, 200 by default.""" def render(self, request: Request) -> bytes: assert request.content body = json.loads(request.content.read().decode()) + request.setResponseCode(getarg(request, b"n", 200, type_=int)) for header_name, header_value in body.items(): request.responseHeaders.setRawHeaders(header_name, [header_value]) return json.dumps(body).encode("utf-8") diff --git a/tests/test_cookies.py b/tests/test_cookies.py deleted file mode 100644 index db78f9f18..000000000 --- a/tests/test_cookies.py +++ /dev/null @@ -1,73 +0,0 @@ -import json - -from twisted.internet import defer -from twisted.trial.unittest import TestCase - -from scrapy.crawler import CrawlerRunner -from scrapy.http import Request -from tests.mockserver import MockServer -from tests.spiders import MetaSpider - - -class CookiesSpider(MetaSpider): - name = "cookies" - - def start_requests(self): - url = self.mockserver.url("/echo2/") - for status in [200, 302, 503]: - # Response params that mockserver will return - body = { - "status": status, - "headers": { - "Set-cookie": f"status={status}", - }, - "body": "", - } - - if status == 302: - body["headers"]["location"] = self.mockserver.url("/") - - yield Request( - url, - callback=self.parse, - meta={"cookiejar": status}, - body=json.dumps(body), - ) - - def parse(self, response): - pass - - def close(self, spider): - mw = [ - m - for m in self.crawler.engine.downloader.middleware.middlewares - if "Cookies" in str(type(m)) - ][0] - jars_data = {} - for k, jar in mw.jars.items(): - jars_data[k] = { - s: v.value for s, v in jar._cookies["127.0.0.1"]["/echo2"].items() - } - self.crawler.stats._stats["cookies_values"] = jars_data - - -class CookiesTestCase(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() - self.runner = CrawlerRunner() - - def tearDown(self): - self.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks - def test_cookies(self): - crawler = CrawlerRunner().create_crawler(CookiesSpider) - yield crawler.crawl(mockserver=self.mockserver) - cookies_stats = crawler.stats._stats.get("cookies_values") - expected_cookies = { - 200: {"status": "200"}, - 302: {"status": "302"}, - 503: {"status": "503"}, - } - self.assertEqual(cookies_stats, expected_cookies) diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py index e44f9bcb8..3f4953447 100644 --- a/tests/utils/bases/download_handlers_http.py +++ b/tests/utils/bases/download_handlers_http.py @@ -284,6 +284,20 @@ class TestHttpBase(ABC): header_value.encode(encoding="utf-8") ] + @coroutine_test + async def test_download_has_correct_response_status_and_headers( + self, mockserver: MockServer + ) -> None: + request = Request( + mockserver.url("/response-headers?n=302", is_secure=self.is_secure), + headers={"content-type": "application/json"}, + body=json.dumps({"Set-Cookie": "status=302"}), + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == 302 + assert response.headers.getlist("Set-Cookie") == [b"status=302"] + @coroutine_test async def test_download_no_extra_response_headers( self, mockserver: MockServer