Removed UrlFilterMiddleware from scrapy.contrib - see this snippet for an alternative: http://snippets.scrapy.org/snippets/12/

docs/topics/spider-middleware.rst

@@ -258,16 +258,6 @@ RequestLimitMiddleware
         upper limit for the amount of requests that can be scheduled per
         domain. Can be set per spider using ``requests_queue_size`` attribute.
 
-UrlFilterMiddleware
--------------------
-
-.. module:: scrapy.contrib.spidermiddleware.urlfilter
-   :synopsis: URL Filter Spider Middleware
-
-.. class:: UrlFilterMiddleware 
-
-   Canonicalizes URLs to filter out duplicated ones
-
 UrlLengthMiddleware
 -------------------
 

scrapy/contrib/spidermiddleware/urlfilter.py

@@ -1,19 +0,0 @@
-"""
-Url Filter Middleware
-
-See documentation in docs/refs/spider-middleware.rst
-"""
-
-from scrapy.http import Request
-from scrapy.utils.url import canonicalize_url
-
-class UrlFilterMiddleware(object):
-    def process_spider_output(self, response, result, spider):
-        disabled = getattr(spider, 'urlfilter_disabled', False)
-        for r in result:
-            if isinstance(r, Request) and not disabled:
-                curl = canonicalize_url(r.url)
-                # only assign if different to avoid re-calculating fingerprint
-                if curl != r.url: 
-                    r.url = curl
-            yield r

scrapy/tests/test_spidermiddleware_urlfilter.py

@@ -1,22 +0,0 @@
-from unittest import TestCase
-
-from scrapy.contrib.spidermiddleware.urlfilter import UrlFilterMiddleware
-from scrapy.http import Response, Request
-from scrapy.spider import BaseSpider
-from scrapy.utils.url import canonicalize_url
-
-
-class TestUrlFilterMiddleware(TestCase):
-
-    def setUp(self):
-        self.spider = BaseSpider('foo')
-        self.mw = UrlFilterMiddleware()
-
-    def test_process_spider_output(self):
-        res = Response('http://scrapytest.org')
-        req_url = 'http://scrapytest.org/?last=1&first=2'
-        reqs = [Request(req_url)]
-
-        out = list(self.mw.process_spider_output(res, reqs, self.spider))
-        self.assertEquals(out[0].url, canonicalize_url(req_url))
-