From 8e8ce301b1a56e40f7e9c322a7b73b8dcfcefc43 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 25 Dec 2018 09:14:09 +0000 Subject: [PATCH] check CONCURRENT_REQUESTS_PER_IP is not set --- scrapy/pqueues.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 75fc198d0..d9effc9d1 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -171,6 +171,14 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): return cls(crawler, qfactory, startprios) def __init__(self, crawler, qfactory, startprios={}): + ip_concurrency_key = 'CONCURRENT_REQUESTS_PER_IP' + ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0) + + if ip_concurrency > 0: + raise ValueError('"%s" does not support %s=%d' % (self.__class__, + ip_concurrency_key, + ip_concurrency)) + super(DownloaderAwarePriorityQueue, self).__init__(qfactory, startprios) self._slots = {slot: 0 for slot in self.pqueues}