core: adapt redirection and media scheduler priorities

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401059

scrapy/trunk/scrapy/contrib/downloadermiddleware/robotstxt.py

@@ -18,6 +18,7 @@ from scrapy.core.exceptions import IgnoreRequest
 from scrapy.conf import settings
 
 class RobotsTxtMiddleware(object):
+    DOWNLOAD_PRIORITY = -1
 
     def __init__(self):
         if not settings.getbool('ROBOTSTXT_OBEY'):
@@ -44,7 +45,7 @@ class RobotsTxtMiddleware(object):
             self._parsers[urldomain] = None
             robotsurl = "%s://%s/robots.txt" % parsedurl[0:2]
             robotsreq = Request(robotsurl)
-            dfd = scrapyengine.schedule(robotsreq, spiders.fromdomain(spiderdomain), priority=0)
+            dfd = scrapyengine.schedule(robotsreq, spiders.fromdomain(spiderdomain), priority=self.DOWNLOAD_PRIORITY)
             dfd.addCallbacks(callback=self._parse_robots, callbackArgs=[urldomain])
             self._spiderdomains[spiderdomain].add(urldomain)
 

scrapy/trunk/scrapy/contrib/pipeline/media.py

@@ -10,6 +10,7 @@ from scrapy.spider import spiders
 
 
 class MediaPipeline(object):
+    DOWNLOAD_PRIORITY = -1
 
     class DomainInfo(object):
         def __init__(self, domain):
@@ -121,7 +122,7 @@ class MediaPipeline(object):
         request fingerprint is used as cache key.
 
         """
-        return scrapyengine.schedule(request, info.spider, priority=0)
+        return scrapyengine.schedule(request, info.spider, priority=self.DOWNLOAD_PRIORITY)
 
     def media_to_download(self, request, info):
         """ Ongoing request hook pre-cache

scrapy/trunk/scrapy/core/engine.py

@@ -42,6 +42,10 @@ class ExecutionEngine(object):
     process for that domain.
     """
 
+    # Scheduler priority of redirected requests. Negative means high priority.
+    # We use high priority to avoid hogging memory with pending redirected requests
+    REDIRECTION_PRIORITY = -10
+
     def __init__(self):
         self.configured = False
         self.keep_alive = False
@@ -356,7 +360,7 @@ class ExecutionEngine(object):
                 return response
             elif isinstance(response, Request):
                 redirected = response # proper alias
-                schd = self.schedule(redirected, spider, priority=-1)
+                schd = self.schedule(redirected, spider, priority=self.REDIRECTION_PRIORITY)
                 chain_deferred(schd, redirected.deferred)
                 return schd