From 1f65d244e50fd574e0d68621179056ac1083685b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 17 Mar 2025 12:26:19 +0100 Subject: [PATCH] Finish the page on optimizations --- docs/conf.py | 1 + docs/topics/downloader-middleware.rst | 2 + docs/topics/optimize.rst | 234 +++++++++----------------- 3 files changed, 85 insertions(+), 152 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 8f706c96c..a7a9bd46a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -154,6 +154,7 @@ intersphinx_mapping = { "tox": ("https://tox.wiki/en/latest/", None), "twisted": ("https://docs.twisted.org/en/stable/", None), "twistedapi": ("https://docs.twisted.org/en/stable/api/", None), + "uvloop": ("https://uvloop.readthedocs.io", None), "w3lib": ("https://w3lib.readthedocs.io/en/latest", None), } intersphinx_disabled_reftypes: Sequence[str] = [] diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 60b6aab78..5ef38a246 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -977,6 +977,8 @@ To retry requests from a spider callback, you can use the .. autofunction:: get_retry_request +.. _retry-settings: + RetryMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/topics/optimize.rst b/docs/topics/optimize.rst index 655415cd4..3dd7282f0 100644 --- a/docs/topics/optimize.rst +++ b/docs/topics/optimize.rst @@ -4,27 +4,14 @@ Optimizations ============= -Scrapy offers different ways to optimize crawls based on :ref:`resource -constraints ` and :ref:`use cases `. - -.. _optimize-resources: - -Lowering resource usage -======================= - -… - -.. - TODO: - Network input and output - optional compression packages - +Scrapy offers different ways to speed up crawls for specific :ref:`use cases +` and to :ref:`lower resource usage `. .. _broad-crawls: .. _topics-broad-crawls: -Optimizing broad crawls -======================= +Speeding up broad crawls +======================== While Scrapy is well suited for **broad crawls**, i.e. crawls that target many websites, the default :ref:`settings ` are optimized for @@ -32,164 +19,107 @@ crawls targetting a single website. For broad crawls, consider these adjustments: -.. _broad-crawls-scheduler-priority-queue: +- .. _broad-crawls-concurrency: -- Set :setting:`SCHEDULER_PRIORITY_QUEUE` to - :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue`. + Increase global concurrency: -.. _broad-crawls-concurrency: + - Set :setting:`CONCURRENT_REQUESTS` as close to “number of target + domains” times :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (e.g. 80 for + 10 domains) as your CPU and memory allow. -Increase concurrency --------------------- + - Increase :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` when increasing + :setting:`CONCURRENT_REQUESTS` stops making a difference. -Concurrency is the number of requests that are processed in parallel. There is -a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that -can be set either per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) or per -IP (:setting:`CONCURRENT_REQUESTS_PER_IP`). + If your CPU or memory become a bottleneck, see :ref:`optimize-resources`. -.. note:: The scheduler priority queue :ref:`recommended for broad crawls - ` does not support - :setting:`CONCURRENT_REQUESTS_PER_IP`. +- Optimize request scheduling: -The default global concurrency limit in Scrapy is not suitable for crawling -many different domains in parallel, so you will want to increase it. How much -to increase it will depend on how much CPU and memory your crawler will have -available. + - .. _broad-crawls-scheduler-priority-queue: -A good starting point is ``100``: + Set :setting:`SCHEDULER_PRIORITY_QUEUE` to + :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue`. -.. code-block:: python + - .. _broad-crawls-bfo: - CONCURRENT_REQUESTS = 100 + If memory is a bottleneck, see if :ref:`crawling in breadth-first order + (BFO) ` lowers memory usage. -But the best way to find out is by doing some trials and identifying at what -concurrency your Scrapy process gets CPU bounded. For optimum performance, you -should pick a concurrency where CPU usage is at 80-90%. +- Improve DNS resolution speed: -Increasing concurrency also increases memory usage. If memory usage is a -concern, you might need to lower your global concurrency limit accordingly. + - Set up your own DNS server, with a local cache and upstream to a `large + DNS server`_, to avoid slowing down your network. + + .. _large DNS server: https://en.wikipedia.org/wiki/Public_recursive_name_server#Notable_public_DNS_service_operators + + - Increase :setting:`REACTOR_THREADPOOL_MAXSIZE` to the minimum value + that avoids DNS resolution timeouts and makes a noticeable positive + impact in crawl speed. + +- Lower the negative impact of some responses: + + - Set :setting:`RETRY_ENABLED` to ``False`` or, if you need retries, + consider lowering :setting:`RETRY_TIMES` and fine-tuning other + :ref:`retry settings `. + + - Lower :setting:`DOWNLOAD_TIMEOUT` to a more reasonable value, to + discard stuck requests more quickly. + + - Set :setting:`REDIRECT_ENABLED` to ``False`` unless you want to follow + redirects. -Increase Twisted IO thread pool maximum size --------------------------------------------- +.. _optimize-resources: -Currently Scrapy does DNS resolution in a blocking way with usage of thread -pool. With higher concurrency levels the crawling could be slow or even fail -hitting DNS resolver timeouts. Possible solution to increase the number of -threads handling DNS queries. The DNS queue will be processed faster speeding -up establishing of connection and crawling overall. +Lowering resource usage +======================= -To increase maximum thread pool size use: +General tips +------------ -.. code-block:: python +- Try :ref:`using the asyncio reactor `, installing + :doc:`uvloop ` and setting :setting:`ASYNCIO_EVENT_LOOP` to + :class:`uvloop.Loop`. - REACTOR_THREADPOOL_MAXSIZE = 20 + Alternatively, try switching :setting:`TWISTED_REACTOR` to :doc:`some other + reactor `. -Setup your own DNS +- Disable unused :ref:`components `. + + For example, set :setting:`COOKIES_ENABLED` to ``False`` unless you need + cookies. + + +Lowering CPU usage ------------------ -If you have multiple crawling processes and single central DNS, it can act -like DoS attack on the DNS server resulting to slow down of entire network or -even blocking your machines. To avoid this setup your own DNS server with -local cache and upstream to some large DNS like OpenDNS or Verizon. - -Reduce log level ----------------- - -When doing broad crawls you are often only interested in the crawl rates you -get and any errors found. These stats are reported by Scrapy when using the -``INFO`` log level. In order to save CPU (and log storage requirements) you -should not use ``DEBUG`` log level when performing large broad crawls in -production. Using ``DEBUG`` level when developing your (broad) crawler may be -fine though. - -To set the log level use: - -.. code-block:: python - - LOG_LEVEL = "INFO" - -Disable cookies ---------------- - -Disable cookies unless you *really* need. Cookies are often not needed when -doing broad crawls (search engine crawlers ignore them), and they improve -performance by saving some CPU cycles and reducing the memory footprint of your -Scrapy crawler. - -To disable cookies use: - -.. code-block:: python - - COOKIES_ENABLED = False - -Disable retries ---------------- - -Retrying failed HTTP requests can slow down the crawls substantially, specially -when sites causes are very slow (or fail) to respond, thus causing a timeout -error which gets retried many times, unnecessarily, preventing crawler capacity -to be reused for other domains. - -To disable retries use: - -.. code-block:: python - - RETRY_ENABLED = False - -Reduce download timeout ------------------------ - -Unless you are crawling from a very slow connection (which shouldn't be the -case for broad crawls) reduce the download timeout so that stuck requests are -discarded quickly and free up capacity to process the next ones. - -To reduce the download timeout use: - -.. code-block:: python - - DOWNLOAD_TIMEOUT = 15 - -Disable redirects ------------------ - -Consider disabling redirects, unless you are interested in following them. When -doing broad crawls it's common to save redirects and resolve them when -revisiting the site at a later crawl. This also help to keep the number of -request constant per crawl batch, otherwise redirect loops may cause the -crawler to dedicate too many resources on any specific domain. - -To disable redirects use: - -.. code-block:: python - - REDIRECT_ENABLED = False - -.. _broad-crawls-bfo: - -Crawl in BFO order ------------------- - -:ref:`Scrapy crawls in DFO order by default `. - -In broad crawls, however, page crawling tends to be faster than page -processing. As a result, unprocessed early requests stay in memory until the -final depth is reached, which can significantly increase memory usage. - -:ref:`Crawl in BFO order ` instead to save memory. +- Set :setting:`LOG_LEVEL` to ``"INFO"`` or higher. -Be mindful of memory leaks --------------------------- +Lowering memory usage +--------------------- -If your broad crawl shows a high memory usage, in addition to :ref:`crawling in -BFO order ` and :ref:`lowering concurrency -` you should :ref:`debug your memory leaks -`. +- If lowering your crawl speed is OK: + + - Increase the :attr:`~scrapy.Request.priority` of requests whose + :attr:`~scrapy.Request.callback` cannot yield additional requests. + + - If you have many seed requests (see + :meth:`~scrapy.Spider.yield_seeds`), set :setting:`SEEDING_POLICY` to + :class:`~scrapy.SeedingPolicy.lazy` or + :class:`~scrapy.SeedingPolicy.idle`. + + - Lower :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE`. + +- Be in the lookout for :ref:`memory leaks `. -Install a specific Twisted reactor ----------------------------------- +Lowering network usage +---------------------- -If the crawl is exceeding the system's capabilities, you might want to try -installing a specific Twisted reactor, via the :setting:`TWISTED_REACTOR` setting. +- Install brotli_ and zstandard_ to support brotli-compressed_ and + zstd-compressed_ responses. + + .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt + .. _brotli: https://pypi.org/project/Brotli/ + .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt + .. _zstandard: https://pypi.org/project/zstandard/