From 079de6771904edabbd75209a61d5277bbe6460bd Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 14 May 2011 21:47:25 -0300 Subject: [PATCH] downloader: minor performance improvement by using collections.deque() to avoid the list.pop(0) call which is O(n) --- scrapy/core/downloader/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 0471ad456..8cd69a7c7 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -4,6 +4,7 @@ Download web pages using asynchronous IO import random from time import time +from collections import deque from twisted.internet import reactor, defer from twisted.python.failure import Failure @@ -31,7 +32,7 @@ class SpiderInfo(object): log.msg(msg, spider=spider) self.spider = spider self.active = set() - self.queue = [] + self.queue = deque() self.transferring = set() self.closing = False self.lastseen = 0 @@ -128,7 +129,7 @@ class Downloader(object): # Process enqueued requests if there are free slots to transfer for this site while site.queue and site.free_transfer_slots() > 0: - request, deferred = site.queue.pop(0) + request, deferred = site.queue.popleft() if site.closing: dfd = defer.fail(Failure(IgnoreRequest())) else: