diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 928cb7b20..46beba2dd 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -79,7 +79,8 @@ class FilesystemCacheStorage(object): body = f.read() with open(join(rpath, 'response_headers'), 'rb') as f: rawheaders = f.read() - url = metadata['url'] + # We failback to metadata['url'] to support old generated caches ' should be removed for Scrapy 0.11 + url = metadata.get('response_url') or metadata['url'] status = metadata['status'] headers = Headers(headers_raw_to_dict(rawheaders)) respcls = responsetypes.from_args(headers=headers, url=url) @@ -95,6 +96,7 @@ class FilesystemCacheStorage(object): 'url': request.url, 'method': request.method, 'status': response.status, + 'response_url': response.url, 'timestamp': time(), } with open(join(rpath, 'meta'), 'wb') as f: diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index 29782674a..a5b6ce02f 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -68,6 +68,17 @@ class HttpCacheMiddlewareTest(unittest.TestCase): self.assertEqualResponse(self.response, response) assert 'cached' in response.flags + def test_different_request_response_urls(self): + mw = HttpCacheMiddleware(self._get_settings()) + req = Request('http://host.com/path') + res = Response('http://host2.net/test.html') + assert mw.process_request(req, self.spider) is None + mw.process_response(req, res, self.spider) + cached = mw.process_request(req, self.spider) + assert isinstance(cached, Response) + self.assertEqualResponse(res, cached) + assert 'cached' in cached.flags + def test_middleware_ignore_missing(self): mw = self._get_middleware(HTTPCACHE_IGNORE_MISSING=True) self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider)