From 7e5e00cea5a9a1e8665cdfc83ef7a20f3b902291 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 18 Jun 2011 02:52:21 -0300 Subject: [PATCH] Added public engine.download() method to use the downloader bypassing the scheduler. Changed media pipeline to use engine.download() to prevent deadlocks. --- scrapy/contrib/pipeline/media.py | 5 ++--- scrapy/core/engine.py | 9 ++++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index c761a8a74..3ab7d8659 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -3,7 +3,7 @@ from twisted.internet.defer import Deferred, DeferredList from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy import log -from scrapy.utils.request import request_fingerprint, request_deferred +from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter class MediaPipeline(object): @@ -75,11 +75,10 @@ class MediaPipeline(object): errback=self.media_failed, errbackArgs=(request, info)) else: request.meta['handle_httpstatus_all'] = True - dfd = request_deferred(request) + dfd = self.crawler.engine.download(request, info.spider) dfd.addCallbacks( callback=self.media_downloaded, callbackArgs=(request, info), errback=self.media_failed, errbackArgs=(request, info)) - self.crawler.engine.crawl(request, info.spider) return dfd def _cache_result_and_execute_waiters(self, result, fp, info): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 4c5f2f701..499c3cf3b 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -97,7 +97,7 @@ class ExecutionEngine(object): request = self.scheduler.next_request(spider) if not request: return - d = self.download(request, spider) + d = self._download(request, spider) d.addBoth(self._handle_downloader_output, request, spider) d.addBoth(lambda _: self.next_request(spider)) return d @@ -160,6 +160,13 @@ class ExecutionEngine(object): return self.scheduler.enqueue_request(spider, request) def download(self, request, spider): + if isinstance(request, Response): + return request + d = self._download(request, spider) + d.addCallback(self.download, spider) + return d + + def _download(self, request, spider): def _on_success(response): """handle the result of a page download""" assert isinstance(response, (Response, Request))