removed obsolete RestrictMiddleware

docs/topics/settings.rst

@@ -858,7 +858,6 @@ Default::
         'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50,
         'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100,
         'scrapy.contrib.spidermiddleware.requestlimit.RequestLimitMiddleware': 200,
-        'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware': 300,
         'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500,
         'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700,
         'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800,

docs/topics/spider-middleware.rst

@@ -243,21 +243,6 @@ RequestLimitMiddleware
         upper limit for the amount of requests that can be scheduled per
         domain. Can be set per spider using ``requests_queue_size`` attribute.
 
-RestrictMiddleware
-------------------
-
-.. module:: scrapy.contrib.spidermiddleware.restrict
-   :synopsis: Restrict Spider Middleware
-
-.. class:: RestrictMiddleware 
-
-   Restricts crawling to fixed set of particular URLs.
-
-   The :class:`RestrictMiddleware` can be configured through the following
-   settings (see the settings documentation for more info):
-
-      * :setting:`RESTRICT_TO_URLS` - Set of URLs allowed to crawl.
-
 UrlFilterMiddleware
 -------------------
 

scrapy/conf/default_settings.py

@@ -154,7 +154,6 @@ SPIDER_MIDDLEWARES_BASE = {
     'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50,
     'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100,
     'scrapy.contrib.spidermiddleware.requestlimit.RequestLimitMiddleware': 200,
-    'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware': 300,
     'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500,
     'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700,
     'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800,

scrapy/contrib/spidermiddleware/restrict.py

@@ -1,22 +0,0 @@
-"""
-Restrict Spider Middleware
-
-See documentation in docs/topics/spider-middleware.rst
-"""
-
-from itertools import ifilter
-
-from scrapy.http import Request
-from scrapy.core.exceptions import NotConfigured
-from scrapy.conf import settings
-
-class RestrictMiddleware(object):
-    def __init__(self):
-        self.allowed_urls = set(settings.getlist('RESTRICT_TO_URLS'))
-        if not self.allowed_urls:
-            raise NotConfigured
-
-    def process_spider_output(self, response, result, spider):
-        return ifilter(lambda r: isinstance(r, Request) \
-            and r.url not in self.allowed_urls, result or ())
-