diff --git a/scrapy/trunk/docs/faq.rst b/scrapy/trunk/docs/faq.rst index 2661f4ff1..4bd3ad8ac 100644 --- a/scrapy/trunk/docs/faq.rst +++ b/scrapy/trunk/docs/faq.rst @@ -70,3 +70,8 @@ How can I simulate a user login in my spider? --------------------------------------------- See :ref:`ref-request-userlogin`. + +Can I crawl in depth-first order instead of breadth-first order? +---------------------------------------------------------------- + +Yes, there's a setting for that: :setting:`SCHEDULER_ORDER`. diff --git a/scrapy/trunk/docs/ref/settings.rst b/scrapy/trunk/docs/ref/settings.rst index e8fc2d52d..ca9d1ec6c 100644 --- a/scrapy/trunk/docs/ref/settings.rst +++ b/scrapy/trunk/docs/ref/settings.rst @@ -702,11 +702,23 @@ The scheduler to use for crawling. .. setting:: SCHEDULER_ORDER +SCHEDULER_ORDER +--------------- + Default: ``'BFO'`` Scope: ``scrapy.core.scheduler`` -The order to use for the crawling scheduler. +The order to use for the crawling scheduler. Available orders are: + +* ``'BFO'``: `Breadth-first order`_ - typically consumes more memory but + reaches most relevant pages earlier. + +* ``'DFO'``: `Depth-first order`_ - typically consumes less memory than + but takes longer to reach most relevant pages. + +.. _Breadth-first order: http://en.wikipedia.org/wiki/Breadth-first_search +.. _Depth-first order: http://en.wikipedia.org/wiki/Depth-first_search .. setting:: SCHEDULER_MIDDLEWARES diff --git a/scrapy/trunk/scrapy/core/scheduler/schedulers.py b/scrapy/trunk/scrapy/core/scheduler/schedulers.py index 689a77855..55251b180 100644 --- a/scrapy/trunk/scrapy/core/scheduler/schedulers.py +++ b/scrapy/trunk/scrapy/core/scheduler/schedulers.py @@ -38,15 +38,6 @@ class Scheduler(object) : necesarily the order you put them in. ``pending_domains_count`` contains the names of all domains that are to be scheduled. - - Two crawling orders are available by default, which can be set with the - SCHEDULER_ORDER settings: - - * BFO - breath-first order (default). Consumes more memory than DFO but reaches - most relevant pages faster. - - * DFO - depth-first order. Consumes less memory than BFO but usually takes - longer to reach the most relevant pages. """ def __init__(self):