Added public engine.download() method to use the downloader bypassing the scheduler. Changed media pipeline to use engine.download() to prevent deadlocks.

scrapy/contrib/pipeline/media.py

@@ -3,7 +3,7 @@ from twisted.internet.defer import Deferred, DeferredList
 
 from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy import log
-from scrapy.utils.request import request_fingerprint, request_deferred
+from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
 
 class MediaPipeline(object):
@@ -75,11 +75,10 @@ class MediaPipeline(object):
                 errback=self.media_failed, errbackArgs=(request, info))
         else:
             request.meta['handle_httpstatus_all'] = True
-            dfd = request_deferred(request)
+            dfd = self.crawler.engine.download(request, info.spider)
             dfd.addCallbacks(
                 callback=self.media_downloaded, callbackArgs=(request, info),
                 errback=self.media_failed, errbackArgs=(request, info))
-            self.crawler.engine.crawl(request, info.spider)
         return dfd
 
     def _cache_result_and_execute_waiters(self, result, fp, info):

scrapy/core/engine.py

@@ -97,7 +97,7 @@ class ExecutionEngine(object):
         request = self.scheduler.next_request(spider)
         if not request:
             return
-        d = self.download(request, spider)
+        d = self._download(request, spider)
         d.addBoth(self._handle_downloader_output, request, spider)
         d.addBoth(lambda _: self.next_request(spider))
         return d
@@ -160,6 +160,13 @@ class ExecutionEngine(object):
         return self.scheduler.enqueue_request(spider, request)
 
     def download(self, request, spider):
+        if isinstance(request, Response):
+            return request
+        d = self._download(request, spider)
+        d.addCallback(self.download, spider)
+        return d
+
+    def _download(self, request, spider):
         def _on_success(response):
             """handle the result of a page download"""
             assert isinstance(response, (Response, Request))