diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 4f7ab594f..f680b9082 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -83,6 +83,7 @@ class Downloader: self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) + self.per_slot_settings = self.settings.getdict('PER_SLOT_SETTINGS', {}) def fetch(self, request, spider): def _deactivate(response): @@ -99,9 +100,13 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + conc = self.per_slot_settings.get(key,{}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - self.slots[key] = Slot(conc, delay, self.randomize_delay) + delay = self.per_slot_settings.get(key,{}).get('delay', delay) + randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) + new_slot = Slot(conc, delay, randomize_delay) + self.slots[key] = new_slot + return key, self.slots[key]