From ce03ccd4ecea6e3d341da23202024053428529e8 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 23 Sep 2011 13:22:25 -0300 Subject: [PATCH] updated documentation about DEPTH_PRIORITY and DFO/BFO crawls --- docs/faq.rst | 13 +++++++++++-- docs/topics/settings.rst | 9 ++------- docs/topics/spider-middleware.rst | 2 +- scrapy/contrib/spidermiddleware/depth.py | 13 +------------ 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 0cd058284..36e06d8b9 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -87,8 +87,14 @@ See :ref:`topics-request-response-ref-request-userlogin`. Does Scrapy crawl in breath-first or depth-first order? ------------------------------------------------------- -It crawls on breath-first order by default, but you can change it to -depth-first order by setting the :setting:`DEPTH_PRIORITY` setting to ``-1``. +By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which +basically means that it crawls in `DFO order`_. This order is more convenient +in most cases. If you do want to crawl in true `BFO order`_, you can do it by +setting the following settings:: + + DEPTH_PRIORITY = 1 + SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleFifoDiskQueue' + SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.FifoMemoryQueue' My Scrapy crawler has memory leaks. What can I do? -------------------------------------------------- @@ -272,3 +278,6 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _Google cache: http://www.googleguide.com/cached_pages.html .. _Tor project: https://www.torproject.org/ .. _commercial support: http://scrapy.org/support/ +.. _LIFO: http://en.wikipedia.org/wiki/LIFO +.. _DFO order: http://en.wikipedia.org/wiki/Depth-first_search +.. _BFO order: http://en.wikipedia.org/wiki/Breadth-first_search diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index bd9dcc1ad..3c7869d7f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -327,16 +327,11 @@ will be imposed. DEPTH_PRIORITY -------------- -Default: ``1`` +Default: ``0`` An integer that is used to adjust the request priority based on its depth. -To crawl in `breadth-first order`_, set :setting:`DEPTH_PRIORITY` to ``1``. - -To crawl in `depth-first order`_, set :setting:`DEPTH_PRIORITY` to ``-1``. - -To disable any priority adjustment based on depth, set -:setting:`DEPTH_PRIORITY` to ``0``. +If zero, no priority adjustment is made from depth. .. setting:: DEPTH_STATS diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index ea3466ecc..2ba0b1d05 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -165,7 +165,7 @@ DepthMiddleware crawl for any site. If zero, no limit will be imposed. * :setting:`DEPTH_STATS` - Whether to collect depth stats. * :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on - their depth, to crawl in breadh-first or depth-first order. + their depth. HttpErrorMiddleware ------------------- diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py index 20390280e..4023b4009 100644 --- a/scrapy/contrib/spidermiddleware/depth.py +++ b/scrapy/contrib/spidermiddleware/depth.py @@ -23,18 +23,7 @@ class DepthMiddleware(object): maxdepth = settings.getint('DEPTH_LIMIT') usestats = settings.getbool('DEPTH_STATS') verbose = settings.getbool('DEPTH_STATS_VERBOSE') - sorder = settings['SCHEDULER_ORDER'] - if sorder: - # XXX: backwards compatibility with old SCHEDULER_ORDER setting - # will be removed on Scrapy 0.15 - warnings.warn("SCHEDULER_ORDER setting is deprecated, " \ - "use DEPTH_PRIORITY instead", ScrapyDeprecationWarning) - if sorder == 'BFO': - prio = 1 - elif sorder == 'DFO': - prio = -1 - else: - prio = settings.getint('DEPTH_PRIORITY') + prio = settings.getint('DEPTH_PRIORITY') if usestats: from scrapy.stats import stats else: