From 861a803cc3cbbd7bff4273ea86697fe59f167da9 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 4 Sep 2009 17:22:56 -0300 Subject: [PATCH] removed obsolete RestrictMiddleware --- docs/topics/settings.rst | 1 - docs/topics/spider-middleware.rst | 15 -------------- scrapy/conf/default_settings.py | 1 - scrapy/contrib/spidermiddleware/restrict.py | 22 --------------------- 4 files changed, 39 deletions(-) delete mode 100644 scrapy/contrib/spidermiddleware/restrict.py diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f1beae3a8..e3c554cb5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -858,7 +858,6 @@ Default:: 'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50, 'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100, 'scrapy.contrib.spidermiddleware.requestlimit.RequestLimitMiddleware': 200, - 'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware': 300, 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500, 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700, 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800, diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a1c2099fb..682f9d696 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -243,21 +243,6 @@ RequestLimitMiddleware upper limit for the amount of requests that can be scheduled per domain. Can be set per spider using ``requests_queue_size`` attribute. -RestrictMiddleware ------------------- - -.. module:: scrapy.contrib.spidermiddleware.restrict - :synopsis: Restrict Spider Middleware - -.. class:: RestrictMiddleware - - Restricts crawling to fixed set of particular URLs. - - The :class:`RestrictMiddleware` can be configured through the following - settings (see the settings documentation for more info): - - * :setting:`RESTRICT_TO_URLS` - Set of URLs allowed to crawl. - UrlFilterMiddleware ------------------- diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 0969bd046..cb0616de6 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -154,7 +154,6 @@ SPIDER_MIDDLEWARES_BASE = { 'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50, 'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100, 'scrapy.contrib.spidermiddleware.requestlimit.RequestLimitMiddleware': 200, - 'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware': 300, 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500, 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700, 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800, diff --git a/scrapy/contrib/spidermiddleware/restrict.py b/scrapy/contrib/spidermiddleware/restrict.py deleted file mode 100644 index 17f6b8113..000000000 --- a/scrapy/contrib/spidermiddleware/restrict.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Restrict Spider Middleware - -See documentation in docs/topics/spider-middleware.rst -""" - -from itertools import ifilter - -from scrapy.http import Request -from scrapy.core.exceptions import NotConfigured -from scrapy.conf import settings - -class RestrictMiddleware(object): - def __init__(self): - self.allowed_urls = set(settings.getlist('RESTRICT_TO_URLS')) - if not self.allowed_urls: - raise NotConfigured - - def process_spider_output(self, response, result, spider): - return ifilter(lambda r: isinstance(r, Request) \ - and r.url not in self.allowed_urls, result or ()) -