RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

url ending fix

This commit is contained in:
Akshay Sharma 2022-06-21 02:44:18 -04:00
parent 3e9efe26ce
commit ef6290ef1b
3 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scrapy/extensions/httpcache.py
View File

@ -241,6 +241,7 @@ class DbmCacheStorage:
headers = Headers(data['headers'])
body = data['body']
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
print(respcls)
response = respcls(url=url, headers=headers, status=status, body=body)
return response

1
scrapy/responsetypes.py
View File

@ -235,6 +235,7 @@ class ResponseTypes:
http_origin = not url or urlparse(url).scheme in ("http", "https")
content_types = (self._guess_content_type(body=body, headers=headers, url=url, filename=filename),)
content_types = None if content_types == (None,) else content_types
print(content_types)
mime_type = extract_mime(body, content_types=content_types, http_origin=http_origin)
return self._guess_response_type(mime_type)

3
tests/test_downloadermiddleware_httpcache.py
View File

@ -128,12 +128,13 @@ class DefaultStorageTest(_BaseTest):
with self._storage() as storage:
assert storage.retrieve_response(self.spider, self.request) is None
response = Response(
'http://www.example.com',
'http://www.example.com/',
body=b'<!DOCTYPE html>\n<title>.</title>',
status=202,
)
storage.store_response(self.spider, self.request, response)
cached_response = storage.retrieve_response(self.spider, self.request)
print(cached_response)
self.assertIsInstance(cached_response, HtmlResponse)
self.assertEqualResponse(response, cached_response)