From 8da9836db3d6071eff6d0267ed47a7af81b830d3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 24 Jun 2015 03:20:46 +0500 Subject: [PATCH 1/2] don't access settings from downloader.Slot --- scrapy/core/downloader/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index ecbf33039..682c95a15 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -17,10 +17,11 @@ from .handlers import DownloadHandlers class Slot(object): """Downloader slot""" - def __init__(self, concurrency, delay, settings): + def __init__(self, concurrency, delay, randomize_delay): self.concurrency = concurrency self.delay = delay - self.randomize_delay = settings.getbool('RANDOMIZE_DOWNLOAD_DELAY') + self.randomize_delay = randomize_delay + self.active = set() self.queue = deque() self.transferring = set() @@ -66,6 +67,7 @@ class Downloader(object): self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS') self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP') + self.randomize_delay = self.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY') self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) @@ -87,7 +89,7 @@ class Downloader(object): if key not in self.slots: conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency conc, delay = _get_concurrency_delay(conc, spider, self.settings) - self.slots[key] = Slot(conc, delay, self.settings) + self.slots[key] = Slot(conc, delay, self.randomize_delay) return key, self.slots[key] From cb0445e8b884d350786fb6df829c9e00ade5036f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 24 Jun 2015 03:22:23 +0500 Subject: [PATCH 2/2] Slot.__str__ and __repr__ methods; remove unused import --- scrapy/core/downloader/__init__.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 682c95a15..965471851 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,6 +1,8 @@ +from __future__ import absolute_import import random import warnings from time import time +from datetime import datetime from collections import deque from twisted.internet import reactor, defer, task @@ -8,7 +10,6 @@ from twisted.internet import reactor, defer, task from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached from scrapy.resolver import dnscache -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy import signals from .middleware import DownloaderMiddlewareManager from .handlers import DownloadHandlers @@ -40,6 +41,21 @@ class Slot(object): if self.latercall and self.latercall.active(): self.latercall.cancel() + def __repr__(self): + cls_name = self.__class__.__name__ + return "%s(concurrency=%r, delay=%0.2f, randomize_delay=%r)" % ( + cls_name, self.concurrency, self.delay, self.randomize_delay) + + def __str__(self): + return ( + "" % ( + self.concurrency, self.delay, self.randomize_delay, + len(self.active), len(self.queue), len(self.transferring), + datetime.fromtimestamp(self.lastseen).isoformat() + ) + ) + def _get_concurrency_delay(concurrency, spider, settings): delay = settings.getfloat('DOWNLOAD_DELAY')