Fixed bug in url_query_cleaner that returned wrong parameters for urls with fragments and added test

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40379

scrapy/trunk/scrapy/tests/test_utils_url.py

@@ -78,6 +78,8 @@ class UrlUtilsTest(unittest.TestCase):
                          'product.html?id=200')
         self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name']),
                          'product.html?id=200&name=wired')
+        self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired#id20", ['id', 'foo']),
+                         'product.html?id=200&foo=bar')
 
     def test_canonicalize_url(self):
         # no query arguments

scrapy/trunk/scrapy/utils/url.py

@@ -87,6 +87,7 @@ def url_query_parameter(url, parameter, default=None, keep_blank_values=0):
 def url_query_cleaner(url, parameterlist=(), sep='&', kvsep='='):
     """Clean url arguments leaving only those passed in the parameterlist"""
     try:
+        url = urlparse.urldefrag(url)[0]
         base, query = url.split('?', 1)
         parameters = [pair.split(kvsep, 1) for pair in query.split(sep)]
     except: