From e9ebebb230e7ecda2f55944bd20ba76940711b02 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Tue, 7 Sep 2010 17:51:02 -0300
Subject: [PATCH] Removed UrlFilterMiddleware from scrapy.contrib - see this
 snippet for an alternative: http://snippets.scrapy.org/snippets/12/

---
 docs/topics/spider-middleware.rst             | 10 ---------
 scrapy/contrib/spidermiddleware/urlfilter.py  | 19 ----------------
 .../tests/test_spidermiddleware_urlfilter.py  | 22 -------------------
 3 files changed, 51 deletions(-)
 delete mode 100644 scrapy/contrib/spidermiddleware/urlfilter.py
 delete mode 100644 scrapy/tests/test_spidermiddleware_urlfilter.py

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 9ba853814..7396e0749 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -258,16 +258,6 @@ RequestLimitMiddleware
         upper limit for the amount of requests that can be scheduled per
         domain. Can be set per spider using ``requests_queue_size`` attribute.
 
-UrlFilterMiddleware
--------------------
-
-.. module:: scrapy.contrib.spidermiddleware.urlfilter
-   :synopsis: URL Filter Spider Middleware
-
-.. class:: UrlFilterMiddleware 
-
-   Canonicalizes URLs to filter out duplicated ones
-
 UrlLengthMiddleware
 -------------------
 
diff --git a/scrapy/contrib/spidermiddleware/urlfilter.py b/scrapy/contrib/spidermiddleware/urlfilter.py
deleted file mode 100644
index e3d30eb6e..000000000
--- a/scrapy/contrib/spidermiddleware/urlfilter.py
+++ /dev/null
@@ -1,19 +0,0 @@
-"""
-Url Filter Middleware
-
-See documentation in docs/refs/spider-middleware.rst
-"""
-
-from scrapy.http import Request
-from scrapy.utils.url import canonicalize_url
-
-class UrlFilterMiddleware(object):
-    def process_spider_output(self, response, result, spider):
-        disabled = getattr(spider, 'urlfilter_disabled', False)
-        for r in result:
-            if isinstance(r, Request) and not disabled:
-                curl = canonicalize_url(r.url)
-                # only assign if different to avoid re-calculating fingerprint
-                if curl != r.url: 
-                    r.url = curl
-            yield r
diff --git a/scrapy/tests/test_spidermiddleware_urlfilter.py b/scrapy/tests/test_spidermiddleware_urlfilter.py
deleted file mode 100644
index 45621599a..000000000
--- a/scrapy/tests/test_spidermiddleware_urlfilter.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from unittest import TestCase
-
-from scrapy.contrib.spidermiddleware.urlfilter import UrlFilterMiddleware
-from scrapy.http import Response, Request
-from scrapy.spider import BaseSpider
-from scrapy.utils.url import canonicalize_url
-
-
-class TestUrlFilterMiddleware(TestCase):
-
-    def setUp(self):
-        self.spider = BaseSpider('foo')
-        self.mw = UrlFilterMiddleware()
-
-    def test_process_spider_output(self):
-        res = Response('http://scrapytest.org')
-        req_url = 'http://scrapytest.org/?last=1&first=2'
-        reqs = [Request(req_url)]
-
-        out = list(self.mw.process_spider_output(res, reqs, self.spider))
-        self.assertEquals(out[0].url, canonicalize_url(req_url))
-