httpcache must restore responses using response.url instead of request.url

--HG--
extra : rebase_source : 08fa2c3862bb35db2234e0f9bb9cb9ce4a8f4d8d

scrapy/contrib/downloadermiddleware/httpcache.py

@@ -79,7 +79,8 @@ class FilesystemCacheStorage(object):
             body = f.read()
         with open(join(rpath, 'response_headers'), 'rb') as f:
             rawheaders = f.read()
-        url = metadata['url']
+        # We failback to metadata['url'] to support old generated caches ' should be removed for Scrapy 0.11
+        url = metadata.get('response_url') or metadata['url']
         status = metadata['status']
         headers = Headers(headers_raw_to_dict(rawheaders))
         respcls = responsetypes.from_args(headers=headers, url=url)
@@ -95,6 +96,7 @@ class FilesystemCacheStorage(object):
             'url': request.url,
             'method': request.method,
             'status': response.status,
+            'response_url': response.url,
             'timestamp': time(),
         }
         with open(join(rpath, 'meta'), 'wb') as f:

scrapy/tests/test_downloadermiddleware_httpcache.py

@@ -68,6 +68,17 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
         self.assertEqualResponse(self.response, response)
         assert 'cached' in response.flags
 
+    def test_different_request_response_urls(self):
+        mw = HttpCacheMiddleware(self._get_settings())
+        req = Request('http://host.com/path')
+        res = Response('http://host2.net/test.html')
+        assert mw.process_request(req, self.spider) is None
+        mw.process_response(req, res, self.spider)
+        cached = mw.process_request(req, self.spider)
+        assert isinstance(cached, Response)
+        self.assertEqualResponse(res, cached)
+        assert 'cached' in cached.flags
+
     def test_middleware_ignore_missing(self):
         mw = self._get_middleware(HTTPCACHE_IGNORE_MISSING=True)
         self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider)