Merge pull request #821 from nramirezuy/httpcache_dont_cache-19-689

[MRG+1] httpcache dont_cache meta #19 #689

docs/topics/downloader-middleware.rst

@@ -328,6 +328,9 @@ HttpCacheMiddleware
     You can change the HTTP cache policy with the :setting:`HTTPCACHE_POLICY`
     setting. Or you can also implement your own policy.
 
+    .. reqmeta:: dont_cache
+
+    You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals `True`.
 
 .. _httpcache-policy-dummy:
 

docs/topics/request-response.rst

@@ -227,6 +227,7 @@ Those are:
 * :reqmeta:`handle_httpstatus_all`
 * ``dont_merge_cookies`` (see ``cookies`` parameter of :class:`Request` constructor)
 * :reqmeta:`cookiejar`
+  :reqmeta:`dont_cache`
 * :reqmeta:`redirect_urls`
 * :reqmeta:`bindaddress`
 * :reqmeta:`dont_obey_robotstxt`

scrapy/contrib/downloadermiddleware/httpcache.py

@@ -28,6 +28,9 @@ class HttpCacheMiddleware(object):
         self.storage.close_spider(spider)
 
     def process_request(self, request, spider):
+        if request.meta.get('dont_cache', False):
+            return
+
         # Skip uncacheable requests
         if not self.policy.should_cache_request(request):
             request.meta['_dont_cache'] = True  # flag as uncacheable
@@ -53,6 +56,9 @@ class HttpCacheMiddleware(object):
         request.meta['cached_response'] = cachedresponse
 
     def process_response(self, request, response, spider):
+        if request.meta.get('dont_cache', False):
+            return response
+
         # Skip cached responses and uncacheable requests
         if 'cached' in response.flags or '_dont_cache' in request.meta:
             request.meta.pop('_dont_cache', None)

tests/test_downloadermiddleware_httpcache.py

@@ -89,6 +89,18 @@ class _BaseTest(unittest.TestCase):
         assert any(h in request2.headers for h in ('If-None-Match', 'If-Modified-Since'))
         self.assertEqual(request1.body, request2.body)
 
+    def test_dont_cache(self):
+        with self._middleware() as mw:
+            self.request.meta['dont_cache'] = True
+            mw.process_response(self.request, self.response, self.spider)
+            self.assertEqual(mw.storage.retrieve_response(self.spider, self.request), None)
+
+        with self._middleware() as mw:
+            self.request.meta['dont_cache'] = False
+            mw.process_response(self.request, self.response, self.spider)
+            if mw.policy.should_cache_response(self.response, self.request):
+                self.assertIsInstance(mw.storage.retrieve_response(self.spider, self.request), self.response.__class__)
+
 
 class DefaultStorageTest(_BaseTest):