core: change default priority to 0 to use balanced priorityqueue, and increase priority of redirected requests so memory doesnt hog because of redirection requests waiting for others requests to finish before they got a chance to be downloaded

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401058

scrapy/trunk/scrapy/core/engine.py

@@ -226,7 +226,7 @@ class ExecutionEngine(object):
     def open_domains(self):
         return self.downloader.sites.keys()
 
-    def crawl(self, request, spider, priority=1, domain_priority=1):
+    def crawl(self, request, spider, priority=0, domain_priority=0):
         domain = spider.domain_name
 
         def _process(response):
@@ -290,7 +290,7 @@ class ExecutionEngine(object):
     def scrape(self, request, response, spider):
         return self.spidermiddleware.scrape(request, response, spider)
 
-    def schedule(self, request, spider, priority=1, domain_priority=1):
+    def schedule(self, request, spider, priority=0, domain_priority=0):
         domain = spider.domain_name
         if not self.scheduler.domain_is_open(domain):
             if self.debug_mode: 
@@ -356,7 +356,7 @@ class ExecutionEngine(object):
                 return response
             elif isinstance(response, Request):
                 redirected = response # proper alias
-                schd = self.schedule(redirected, spider, priority=4)
+                schd = self.schedule(redirected, spider, priority=-1)
                 chain_deferred(schd, redirected.deferred)
                 return schd
 

scrapy/trunk/scrapy/core/scheduler/schedulers.py

@@ -79,7 +79,7 @@ class Scheduler(object) :
             return domain
         return None
 
-    def add_domain(self, domain, priority=1):
+    def add_domain(self, domain, priority=0):
         """This functions schedules a new domain to be scraped, with the given priority.
 
         It doesn't check if the domain is already scheduled.
@@ -99,7 +99,7 @@ class Scheduler(object) :
         Priority = PriorityStack if self.dfo else PriorityQueue
         self.pending_requests[domain] = Priority()
 
-    def enqueue_request(self, domain, request, priority=1):
+    def enqueue_request(self, domain, request, priority=0):
         """Enqueue a request to be downloaded for a domain that is currently being scraped."""
         dfd = defer.Deferred()
         self.pending_requests[domain].push((request, dfd), priority)