From c9cafd5c43a0446ba122772ab5986971a1600132 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Mon, 1 Sep 2008 04:16:51 +0000
Subject: [PATCH] added UrlFilterMiddleware

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40191
---
 .../contrib/spidermiddleware/urlfilter.py       | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py

diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py
new file mode 100644
index 000000000..1034f747d
--- /dev/null
+++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py
@@ -0,0 +1,17 @@
+"""
+UrlFilterMiddleware: canonicalizes URLs to filter out duplicated ones
+"""
+
+from scrapy.http import Request
+from scrapy.utils.url import canonicalize_url
+
+class UrlFilterMiddleware(object):
+    def process_result(self, response, result, spider):
+        disabled = getattr(spider, 'urlfilter_disabled', False)
+        for r in result:
+            if isinstance(r, Request) and not disabled:
+                curl = canonicalize_url(r.url)
+                # only assign if different to avoid re-calculating fingerprint
+                if curl != r.url: 
+                    r.url = curl
+            yield r