From a4b728f2b28d883239e05b5aaeafcfe1dd6aea9f Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 9 Jul 2009 10:55:30 -0300 Subject: [PATCH] Scraper: added lower limit for responses sizes, removed redundant line --- scrapy/core/scraper.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 79b1786b7..a84bcb7b8 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -18,7 +18,7 @@ from scrapy.conf import settings class SiteInfo(object): """Object for holding data of the responses being scraped""" - FAILURE_SIZE = 1024 # make failures equivalent to 1K responses in size + MIN_RESPONSE_SIZE = 1024 def __init__(self, max_active_size=5000000): self.max_active_size = max_active_size @@ -31,9 +31,9 @@ class SiteInfo(object): deferred = defer.Deferred() self.queue.append((response, request, deferred)) if isinstance(response, Response): - self.active_size += len(response.body) + self.active_size += max(len(response.body), self.MIN_RESPONSE_SIZE) else: - self.active_size += self.FAILURE_SIZE + self.active_size += self.MIN_RESPONSE_SIZE return deferred def next_response_request_deferred(self): @@ -44,9 +44,9 @@ class SiteInfo(object): def finish_response(self, response): self.active.remove(response) if isinstance(response, Response): - self.active_size -= len(response.body) + self.active_size -= max(len(response.body), self.MIN_RESPONSE_SIZE) else: - self.active_size -= self.FAILURE_SIZE + self.active_size -= self.MIN_RESPONSE_SIZE def is_idle(self): return not (self.queue or self.active) @@ -133,7 +133,6 @@ class Scraper(object): _failure.value.__class__.__name__)) def handle_spider_output(self, result, request, response, spider): - domain = spider.domain_name if not result: return defer_succeed(None) dfd = parallel(iter(result), self.concurrent_items,