From e6272e5209de38e2632d63ccb39e645b048c97b3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 26 Jun 2015 18:44:23 +0500 Subject: [PATCH 1/6] make AutoThrottle._adjust_delay easier to understand --- scrapy/extensions/throttle.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index a60b0cd83..8c85d13f9 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -68,13 +68,22 @@ class AutoThrottle(object): def _adjust_delay(self, slot, latency, response): """Define delay adjustment policy""" + + # Adjust the delay to be closer to latency. + new_delay = (slot.delay + latency) / 2.0 + # If latency is bigger than old delay, then use latency instead of mean. - # It works better with problematic sites - new_delay = min(max(self.mindelay, latency, (slot.delay + latency) / 2.0), self.maxdelay) + # It works better with problematic sites. + new_delay = max(latency, new_delay) + + # Make sure self.mindelay <= new_delay <= self.max_delay + new_delay = min(max(self.mindelay, new_delay), self.maxdelay) # Dont adjust delay if response status != 200 and new delay is smaller # than old one, as error pages (and redirections) are usually small and # so tend to reduce latency, thus provoking a positive feedback by # reducing delay instead of increase. - if response.status == 200 or new_delay > slot.delay: - slot.delay = new_delay + if response.status != 200 and new_delay <= slot.delay: + return + + slot.delay = new_delay From 584252e8f2f8577d37d648042d0a78a41af8e913 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 26 Jun 2015 18:58:29 +0500 Subject: [PATCH 2/6] move AutoThrottle default options to default_settings.py --- scrapy/extensions/throttle.py | 4 ++-- scrapy/settings/default_settings.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 8c85d13f9..9dceaf00d 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -33,10 +33,10 @@ class AutoThrottle(object): s.getfloat('DOWNLOAD_DELAY') def _max_delay(self, spider): - return self.crawler.settings.getfloat('AUTOTHROTTLE_MAX_DELAY', 60.0) + return self.crawler.settings.getfloat('AUTOTHROTTLE_MAX_DELAY') def _start_delay(self, spider): - return max(self.mindelay, self.crawler.settings.getfloat('AUTOTHROTTLE_START_DELAY', 5.0)) + return max(self.mindelay, self.crawler.settings.getfloat('AUTOTHROTTLE_START_DELAY')) def _response_downloaded(self, response, request, spider): key, slot = self._get_slot(request, spider) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 0ea014007..6df3925e5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -20,6 +20,11 @@ from os.path import join, abspath, dirname AJAXCRAWL_ENABLED = False +AUTOTHROTTLE_ENABLED = False +AUTOTHROTTLE_DEBUG = False +AUTOTHROTTLE_MAX_DELAY = 60.0 +AUTOTHROTTLE_START_DELAY = 5.0 + BOT_NAME = 'scrapybot' CLOSESPIDER_TIMEOUT = 0 From 17cb51315b875157d397719c2834d1d02f2ca49f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 26 Jun 2015 19:07:54 +0500 Subject: [PATCH 3/6] drop support for AUTOTHROTTLE_MIN_DOWNLOAD_DELAY it was deprecated for years --- scrapy/extensions/throttle.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 9dceaf00d..46273353b 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -28,9 +28,7 @@ class AutoThrottle(object): def _min_delay(self, spider): s = self.crawler.settings - return getattr(spider, 'download_delay', 0.0) or \ - s.getfloat('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY') or \ - s.getfloat('DOWNLOAD_DELAY') + return getattr(spider, 'download_delay', 0.0) or s.getfloat('DOWNLOAD_DELAY') def _max_delay(self, spider): return self.crawler.settings.getfloat('AUTOTHROTTLE_MAX_DELAY') From 1fef9f113237630664ef05645127ca50662d71b6 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 26 Jun 2015 19:09:26 +0500 Subject: [PATCH 4/6] AutoThrottle: respect download_delay=0 spider attribute --- scrapy/extensions/throttle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 46273353b..d2ff8665e 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -28,7 +28,7 @@ class AutoThrottle(object): def _min_delay(self, spider): s = self.crawler.settings - return getattr(spider, 'download_delay', 0.0) or s.getfloat('DOWNLOAD_DELAY') + return getattr(spider, 'download_delay', s.getfloat('DOWNLOAD_DELAY')) def _max_delay(self, spider): return self.crawler.settings.getfloat('AUTOTHROTTLE_MAX_DELAY') From 63317531f958cc36000f527d352ee0b7453653cb Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 26 Jun 2015 20:47:58 +0500 Subject: [PATCH 5/6] DOC fix authrottle docs see https://github.com/scrapy/scrapy/pull/502/files#r8574692 --- docs/topics/autothrottle.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 8073ec6e0..c1a7be6ff 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -33,18 +33,18 @@ server) is, and this extension builds on that premise. Throttling algorithm ==================== -This adjusts download delays and concurrency based on the following rules: +This adjusts download delays based on the following rules: -1. spiders always start with one concurrent request and a download delay of - :setting:`AUTOTHROTTLE_START_DELAY` +1. spiders always start with a download delay of + :setting:`AUTOTHROTTLE_START_DELAY`; 2. when a response is received, the download delay is adjusted to the average of previous download delay and the latency of the response. .. note:: The AutoThrottle extension honours the standard Scrapy settings for - concurrency and delay. This means that it will never set a download delay - lower than :setting:`DOWNLOAD_DELAY` or a concurrency higher than - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` - (or :setting:`CONCURRENT_REQUESTS_PER_IP`, depending on which one you use). + concurrency and delay. This means that it will respect + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and + :setting:`CONCURRENT_REQUESTS_PER_IP` options and + never set a download delay lower than :setting:`DOWNLOAD_DELAY`. Settings ======== From d850238c2220e2d082ea9805fa356b62d975697f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sat, 27 Jun 2015 04:59:42 +0500 Subject: [PATCH 6/6] add AUTOTHROTTLE_TARGET_CONCURRENCY option and expand AutoThrottle docs --- docs/topics/autothrottle.rst | 102 ++++++++++++++++++++++------ docs/topics/settings.rst | 11 +-- scrapy/extensions/throttle.py | 14 ++-- scrapy/settings/default_settings.py | 1 + 4 files changed, 100 insertions(+), 28 deletions(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index c1a7be6ff..0d664cf67 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -12,13 +12,65 @@ Design goals 1. be nicer to sites instead of using default download delay of zero 2. automatically adjust scrapy to the optimum crawling speed, so the user - doesn't have to tune the download delays and concurrent requests to find the - optimum one. The user only needs to specify the maximum concurrent requests + doesn't have to tune the download delays to find the optimum one. + The user only needs to specify the maximum concurrent requests it allows, and the extension does the rest. +.. _autothrottle-algorithm: + How it works ============ +AutoThrottle extension adjusts download delays dynamically to make spider send +:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent requests on average +to each remote website. + +It uses download latency to compute the delays. The main idea is the +following: if a server needs ``latency`` seconds to respond, a client +should send a request each ``latency/N`` seconds to have ``N`` requests +processed in parallel. + +Instead of adjusting the delays one can just set a small fixed +download delay and impose hard limits on concurrency using +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or +:setting:`CONCURRENT_REQUESTS_PER_IP` options. It will provide a similar +effect, but there are some important differences: + +* because the download delay is small there will be occasional bursts + of requests; +* often non-200 (error) responses can be returned faster than regular + responses, so with a small download delay and a hard concurrency limit + crawler will be sending requests to server faster when server starts to + return errors. But this is an opposite of what crawler should do - in case + of errors it makes more sense to slow down: these errors may be caused by + the high request rate. + +AutoThrottle doesn't have these issues. + +Throttling algorithm +==================== + +AutoThrottle algorithm adjusts download delays based on the following rules: + +1. spiders always start with a download delay of + :setting:`AUTOTHROTTLE_START_DELAY`; +2. when a response is received, the target download delay is calculated as + ``latency / N`` where ``latency`` is a latency of the response, + and ``N`` is :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`. +3. download delay for next requests is set to the average of previous + download delay and the target download delay; +4. latencies of non-200 responses are not allowed to decrease the delay; +5. download delay can't become less than :setting:`DOWNLOAD_DELAY` or greater + than :setting:`AUTOTHROTTLE_MAX_DELAY` + +.. note:: The AutoThrottle extension honours the standard Scrapy settings for + concurrency and delay. This means that it will respect + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and + :setting:`CONCURRENT_REQUESTS_PER_IP` options and + never set a download delay lower than :setting:`DOWNLOAD_DELAY`. + +.. _download-latency: + In Scrapy, the download latency is measured as the time elapsed between establishing the TCP connection and receiving the HTTP headers. @@ -28,24 +80,6 @@ callback, for example, and unable to attend downloads. However, these latencies should still give a reasonable estimate of how busy Scrapy (and ultimately, the server) is, and this extension builds on that premise. -.. _autothrottle-algorithm: - -Throttling algorithm -==================== - -This adjusts download delays based on the following rules: - -1. spiders always start with a download delay of - :setting:`AUTOTHROTTLE_START_DELAY`; -2. when a response is received, the download delay is adjusted to the - average of previous download delay and the latency of the response. - -.. note:: The AutoThrottle extension honours the standard Scrapy settings for - concurrency and delay. This means that it will respect - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and - :setting:`CONCURRENT_REQUESTS_PER_IP` options and - never set a download delay lower than :setting:`DOWNLOAD_DELAY`. - Settings ======== @@ -88,6 +122,34 @@ Default: ``60.0`` The maximum download delay (in seconds) to be set in case of high latencies. +.. setting:: AUTOTHROTTLE_TARGET_CONCURRENCY + +AUTOTHROTTLE_TARGET_CONCURRENCY +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: ``1.0`` + +Average number of requests Scrapy should be sending in parallel to remote +websites. + +By default, AutoThrottle adjusts the delay to send a single +concurrent request to each of the remote websites. Set this option to +a higher value (e.g. ``2.0``) to increase the throughput and the load on remote +servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value +(e.g. ``0.5``) makes the crawler more conservative and polite. + +Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` +and :setting:`CONCURRENT_REQUESTS_PER_IP` options are still respected +when AutoThrottle extension is enabled. This means that if +``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or +:setting:`CONCURRENT_REQUESTS_PER_IP`, the crawler won't reach this number +of concurrent requests. + +At every given time point Scrapy can be sending more or less concurrent +requests than ``AUTOTHROTTLE_TARGET_CONCURRENCY``; it is a suggested +value the crawler tries to approach, not a hard limit. + .. setting:: AUTOTHROTTLE_DEBUG AUTOTHROTTLE_DEBUG diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a9eba41ce..2b983f9d9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -187,7 +187,6 @@ Default: ``16`` The maximum number of concurrent (ie. simultaneous) requests that will be performed by the Scrapy downloader. - .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN CONCURRENT_REQUESTS_PER_DOMAIN @@ -198,6 +197,10 @@ Default: ``8`` The maximum number of concurrent (ie. simultaneous) requests that will be performed to any single domain. +See also: :ref:`topics-autothrottle` and its +:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option. + + .. setting:: CONCURRENT_REQUESTS_PER_IP CONCURRENT_REQUESTS_PER_IP @@ -211,9 +214,9 @@ performed to any single IP. If non-zero, the used instead. In other words, concurrency limits will be applied per IP, not per domain. -This setting also affects :setting:`DOWNLOAD_DELAY`: -if :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, download delay is -enforced per IP, not per domain. +This setting also affects :setting:`DOWNLOAD_DELAY` and +:ref:`topics-autothrottle`: if :setting:`CONCURRENT_REQUESTS_PER_IP` +is non-zero, download delay is enforced per IP, not per domain. .. setting:: DEFAULT_ITEM_CLASS diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index d2ff8665e..198d4bbb0 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -14,6 +14,7 @@ class AutoThrottle(object): raise NotConfigured self.debug = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") + self.target_concurrency = crawler.settings.getfloat("AUTOTHROTTLE_TARGET_CONCURRENCY") crawler.signals.connect(self._spider_opened, signal=signals.spider_opened) crawler.signals.connect(self._response_downloaded, signal=signals.response_downloaded) @@ -67,12 +68,17 @@ class AutoThrottle(object): def _adjust_delay(self, slot, latency, response): """Define delay adjustment policy""" - # Adjust the delay to be closer to latency. - new_delay = (slot.delay + latency) / 2.0 + # If a server needs `latency` seconds to respond then + # we should send a request each `latency/N` seconds + # to have N requests processed in parallel + target_delay = latency / self.target_concurrency - # If latency is bigger than old delay, then use latency instead of mean. + # Adjust the delay to make it closer to target_delay + new_delay = (slot.delay + target_delay) / 2.0 + + # If target delay is bigger than old delay, then use it instead of mean. # It works better with problematic sites. - new_delay = max(latency, new_delay) + new_delay = max(target_delay, new_delay) # Make sure self.mindelay <= new_delay <= self.max_delay new_delay = min(max(self.mindelay, new_delay), self.maxdelay) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 6df3925e5..62bb1ccd3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -24,6 +24,7 @@ AUTOTHROTTLE_ENABLED = False AUTOTHROTTLE_DEBUG = False AUTOTHROTTLE_MAX_DELAY = 60.0 AUTOTHROTTLE_START_DELAY = 5.0 +AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 BOT_NAME = 'scrapybot'