RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

per_slot_settings(docs): code style fix (from previous pre-commit check)

This commit is contained in:
Georgiy Zatserklianyi 2023-02-16 17:55:59 +02:00
parent 07e1429877
commit 045092e8d7
3 changed files with 27 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
docs/topics/settings.rst
View File

@ -783,15 +783,8 @@ Allows to define concurrency/delay parameters on per slot(domain) basis:
.. code-block:: python
DOWNLOAD_SLOTS = {
'quotes.toscrape.com': {
'concurrency': 1,
'delay': 2,
'randomize_delay': False
},
'books.toscrape.com': {
'delay': 3,
'randomize_delay': False
}
"quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
"books.toscrape.com": {"delay": 3, "randomize_delay": False},
}
.. note::

9
scrapy/core/downloader/__init__.py
View File

@ -103,13 +103,14 @@ class Downloader:
if key not in self.slots:
slot_settings = self.per_slot_settings.get(key, {})
conc = (
self.ip_concurrency if self.ip_concurrency else self.domain_concurrency)
self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
)
conc, delay = _get_concurrency_delay(conc, spider, self.settings)
conc, delay = (
slot_settings.get('concurrency', conc),
slot_settings.get('delay', delay)
slot_settings.get("concurrency", conc),
slot_settings.get("delay", delay),
)
randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay)
randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
new_slot = Slot(conc, delay, randomize_delay)
self.slots[key] = new_slot

37
tests/test_downloaderslotssettings.py
View File

@ -1,31 +1,31 @@
import time
from twisted.internet import defer
from twisted.trial.unittest import TestCase
from scrapy.crawler import CrawlerRunner
from scrapy.http import Request
from tests.mockserver import MockServer
from tests.spiders import MetaSpider
from twisted.internet import defer
from twisted.trial.unittest import TestCase
class DownloaderSlotsSettingsTestSpider(MetaSpider):
name = 'downloader_slots'
name = "downloader_slots"
custom_settings = {
"DOWNLOAD_DELAY": 1,
"RANDOMIZE_DOWNLOAD_DELAY": False,
"DOWNLOAD_SLOTS": {
'quotes.toscrape.com': {
'concurrency': 1,
'delay': 2,
'randomize_delay': False
"quotes.toscrape.com": {
"concurrency": 1,
"delay": 2,
"randomize_delay": False
},
'books.toscrape.com': {
'delay': 3,
'randomize_delay': False
"books.toscrape.com": {
"delay": 3,
"randomize_delay": False
}
}
}
@ -33,21 +33,21 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):
def start_requests(self):
self.times = {None: []}
slots = list(self.custom_settings.get('DOWNLOAD_SLOTS', {}).keys()) + [None]
slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None]
for slot in slots:
url = self.mockserver.url(f"/?downloader_slot={slot}")
self.times[slot] = []
yield Request(url, callback=self.parse, meta={'download_slot': slot})
yield Request(url, callback=self.parse, meta={"download_slot": slot})
def parse(self, response):
slot = response.meta.get('download_slot', None)
slot = response.meta.get("download_slot", None)
self.times[slot].append(time.time())
url = self.mockserver.url(f"/?downloader_slot={slot}&req=2")
yield Request(url, callback=self.not_parse, meta={'download_slot': slot})
yield Request(url, callback=self.not_parse, meta={"download_slot": slot})
def not_parse(self, response):
slot = response.meta.get('download_slot', None)
slot = response.meta.get("download_slot", None)
self.times[slot].append(time.time())
@ -70,6 +70,9 @@ class CrawlTestCase(TestCase):
tolerance = 0.3
delays_real = {k: v[1] - v[0] for k, v in times.items()}
error_delta = {k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) for k, v in slots.items()}
error_delta = {
k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay)
for k, v in slots.items()
}
self.assertTrue(max(list(error_delta.values())) < tolerance)