RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

per slot settings: logging updated (create/close Slot)

This commit is contained in:
GeorgeA92 2021-11-20 21:05:24 +02:00
parent d5acf88ca5
commit 1b7d7ecfcd
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/core/downloader/__init__.py
View File

@ -108,7 +108,8 @@ class Downloader:
randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay)
new_slot = Slot(conc, delay, randomize_delay) new_slot = Slot(conc, delay, randomize_delay)
self.slots[key] = new_slot self.slots[key] = new_slot
logger.debug(f"Downloader slot created {'from per slot settings' if key in self.per_slot_settings.keys() else ''}: {new_slot}") logger.debug(
f"Downloader slot '{key}' - created {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''}: {new_slot}")
return key, self.slots[key] return key, self.slots[key]
@ -205,4 +206,7 @@ class Downloader:
mintime = time() - age mintime = time() - age
for key, slot in list(self.slots.items()): for key, slot in list(self.slots.items()):
if not slot.active and slot.lastseen + slot.delay < mintime: if not slot.active and slot.lastseen + slot.delay < mintime:
self.slots.pop(key).close() inactive_slot = self.slots.pop(key)
inactive_slot.close()
logger.debug(
f"Downloader slot '{key}' - closed {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''} : {inactive_slot}")