From fae3d7a8dc000927fc305c3bcdc5a1b5912e760e Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Mon, 13 Apr 2009 20:27:35 +0000 Subject: [PATCH] core: adapt redirection and media scheduler priorities --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401059 --- .../trunk/scrapy/contrib/downloadermiddleware/robotstxt.py | 3 ++- scrapy/trunk/scrapy/contrib/pipeline/media.py | 3 ++- scrapy/trunk/scrapy/core/engine.py | 6 +++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/robotstxt.py index 99d8e400d..0ae8bcdcd 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -18,6 +18,7 @@ from scrapy.core.exceptions import IgnoreRequest from scrapy.conf import settings class RobotsTxtMiddleware(object): + DOWNLOAD_PRIORITY = -1 def __init__(self): if not settings.getbool('ROBOTSTXT_OBEY'): @@ -44,7 +45,7 @@ class RobotsTxtMiddleware(object): self._parsers[urldomain] = None robotsurl = "%s://%s/robots.txt" % parsedurl[0:2] robotsreq = Request(robotsurl) - dfd = scrapyengine.schedule(robotsreq, spiders.fromdomain(spiderdomain), priority=0) + dfd = scrapyengine.schedule(robotsreq, spiders.fromdomain(spiderdomain), priority=self.DOWNLOAD_PRIORITY) dfd.addCallbacks(callback=self._parse_robots, callbackArgs=[urldomain]) self._spiderdomains[spiderdomain].add(urldomain) diff --git a/scrapy/trunk/scrapy/contrib/pipeline/media.py b/scrapy/trunk/scrapy/contrib/pipeline/media.py index 2e1a025a7..c61c6169a 100644 --- a/scrapy/trunk/scrapy/contrib/pipeline/media.py +++ b/scrapy/trunk/scrapy/contrib/pipeline/media.py @@ -10,6 +10,7 @@ from scrapy.spider import spiders class MediaPipeline(object): + DOWNLOAD_PRIORITY = -1 class DomainInfo(object): def __init__(self, domain): @@ -121,7 +122,7 @@ class MediaPipeline(object): request fingerprint is used as cache key. """ - return scrapyengine.schedule(request, info.spider, priority=0) + return scrapyengine.schedule(request, info.spider, priority=self.DOWNLOAD_PRIORITY) def media_to_download(self, request, info): """ Ongoing request hook pre-cache diff --git a/scrapy/trunk/scrapy/core/engine.py b/scrapy/trunk/scrapy/core/engine.py index 8c47ee3a1..a1f3386b6 100644 --- a/scrapy/trunk/scrapy/core/engine.py +++ b/scrapy/trunk/scrapy/core/engine.py @@ -42,6 +42,10 @@ class ExecutionEngine(object): process for that domain. """ + # Scheduler priority of redirected requests. Negative means high priority. + # We use high priority to avoid hogging memory with pending redirected requests + REDIRECTION_PRIORITY = -10 + def __init__(self): self.configured = False self.keep_alive = False @@ -356,7 +360,7 @@ class ExecutionEngine(object): return response elif isinstance(response, Request): redirected = response # proper alias - schd = self.schedule(redirected, spider, priority=-1) + schd = self.schedule(redirected, spider, priority=self.REDIRECTION_PRIORITY) chain_deferred(schd, redirected.deferred) return schd