From ef6290ef1b955cbe06a47053eda4a8064521713c Mon Sep 17 00:00:00 2001 From: Akshay Sharma <42249933+akshaysharmajs@users.noreply.github.com> Date: Tue, 21 Jun 2022 02:44:18 -0400 Subject: [PATCH] url ending fix --- scrapy/extensions/httpcache.py | 1 + scrapy/responsetypes.py | 1 + tests/test_downloadermiddleware_httpcache.py | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 843e14812..7733ffa5c 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -241,6 +241,7 @@ class DbmCacheStorage: headers = Headers(data['headers']) body = data['body'] respcls = responsetypes.from_args(headers=headers, url=url, body=body) + print(respcls) response = respcls(url=url, headers=headers, status=status, body=body) return response diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 4140d9de0..95252498b 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -235,6 +235,7 @@ class ResponseTypes: http_origin = not url or urlparse(url).scheme in ("http", "https") content_types = (self._guess_content_type(body=body, headers=headers, url=url, filename=filename),) content_types = None if content_types == (None,) else content_types + print(content_types) mime_type = extract_mime(body, content_types=content_types, http_origin=http_origin) return self._guess_response_type(mime_type) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 928c007f5..c84c2bdeb 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -128,12 +128,13 @@ class DefaultStorageTest(_BaseTest): with self._storage() as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response( - 'http://www.example.com', + 'http://www.example.com/', body=b'\n.', status=202, ) storage.store_response(self.spider, self.request, response) cached_response = storage.retrieve_response(self.spider, self.request) + print(cached_response) self.assertIsInstance(cached_response, HtmlResponse) self.assertEqualResponse(response, cached_response)