From 0256a34f9788dd00f46efcef0047dd86c280cd6c Mon Sep 17 00:00:00 2001 From: Artem Bogomyagkov Date: Fri, 5 Oct 2012 19:17:59 +0300 Subject: [PATCH] fix for request's errback, allows to get failed requests correctly --- scrapy/core/engine.py | 6 +----- scrapy/core/scraper.py | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index a70f26449..c63dfdbe2 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -200,16 +200,12 @@ class ExecutionEngine(object): response=response, request=request, spider=spider) return response - def _on_error(failure): - failure.request = request - return failure - def _on_complete(_): slot.nextcall.schedule() return _ dwld = self.downloader.fetch(request, spider) - dwld.addCallbacks(_on_success, _on_error) + dwld.addCallbacks(_on_success) dwld.addBoth(_on_complete) return dwld diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 63d47c99f..75090b258 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -136,6 +136,7 @@ class Scraper(object): request_result, request, spider) def call_spider(self, result, request, spider): + result.request = request dfd = defer_result(result) dfd.addCallbacks(request.callback or spider.parse, request.errback) return dfd.addCallback(iterate_spider_output)