From 7c2476bb25fbfd40118d660a64fb2d81c5e14cd1 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 12 Jun 2009 08:31:30 -0300 Subject: [PATCH] fixed a couple of bugs caused by adding priority to Requests (thanks Artem for reporting) --- scrapy/contrib/downloadermiddleware/robotstxt.py | 4 ++-- scrapy/contrib/pipeline/media.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index 0ae8bcdcd..4e634eaa1 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -44,8 +44,8 @@ class RobotsTxtMiddleware(object): else: self._parsers[urldomain] = None robotsurl = "%s://%s/robots.txt" % parsedurl[0:2] - robotsreq = Request(robotsurl) - dfd = scrapyengine.schedule(robotsreq, spiders.fromdomain(spiderdomain), priority=self.DOWNLOAD_PRIORITY) + robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY) + dfd = scrapyengine.schedule(robotsreq, spiders.fromdomain(spiderdomain)) dfd.addCallbacks(callback=self._parse_robots, callbackArgs=[urldomain]) self._spiderdomains[spiderdomain].add(urldomain) diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index c61c6169a..a3ab25fc0 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -122,7 +122,8 @@ class MediaPipeline(object): request fingerprint is used as cache key. """ - return scrapyengine.schedule(request, info.spider, priority=self.DOWNLOAD_PRIORITY) + request.priority = self.DOWNLOAD_PRIORITY + return scrapyengine.schedule(request, info.spider) def media_to_download(self, request, info): """ Ongoing request hook pre-cache