diff --git a/docs/ref/index.rst b/docs/ref/index.rst index dad425d7f..0c394fb51 100644 --- a/docs/ref/index.rst +++ b/docs/ref/index.rst @@ -19,6 +19,7 @@ This section documents the Scrapy |version| API. For more information see :ref:` extension-manager extensions downloader-middleware + scheduler-middleware link-extractors * :ref:`topics-stats-api` diff --git a/docs/ref/scheduler-middleware.rst b/docs/ref/scheduler-middleware.rst new file mode 100644 index 000000000..930618fb4 --- /dev/null +++ b/docs/ref/scheduler-middleware.rst @@ -0,0 +1,30 @@ +.. _ref-scheduler-middleware: + +======================================== +Built-in scheduler middleware reference +======================================== + +This page describes all scheduler middleware components that come with +Scrapy. + +For a list of the components enabled by default (and their orders) see the +:setting:`SCHEDULER_MIDDLEWARES_BASE` setting. + +Available downloader middlewares +================================ + +DuplicatesFilterMiddleware +-------------------------- + +.. module:: scrapy.contrib.schedulermiddleware.duplicatesfilter + +.. class:: DuplicatesFilterMiddleware + + Filter out already visited urls. + + The :class:`DuplicatesFilterMiddleware` can be configured through the following + settings (see the settings documentation for more info): + + * :setting:`DUPEFILTER_CLASS` - The class used to detect and filter + duplicate requests. + diff --git a/scrapy/contrib/schedulermiddleware/duplicatesfilter.py b/scrapy/contrib/schedulermiddleware/duplicatesfilter.py index c35037d4f..c0ac145a3 100644 --- a/scrapy/contrib/schedulermiddleware/duplicatesfilter.py +++ b/scrapy/contrib/schedulermiddleware/duplicatesfilter.py @@ -1,5 +1,7 @@ """ DuplicatesFilterMiddleware: Filter out already visited urls + +See documentation in docs/ref/scheduler-middleware.rst """ from scrapy.core.exceptions import IgnoreRequest, NotConfigured @@ -7,7 +9,7 @@ from scrapy.utils.misc import load_object from scrapy.conf import settings class DuplicatesFilterMiddleware(object): - """Filter out already seen requests to avoid visiting pages more than once.""" + def __init__(self): clspath = settings.get('DUPEFILTER_CLASS') if not clspath: