diff --git a/scrapy/trunk/scrapy/tests/test_utils_url.py b/scrapy/trunk/scrapy/tests/test_utils_url.py index bc7dae523..39687c313 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_url.py +++ b/scrapy/trunk/scrapy/tests/test_utils_url.py @@ -78,6 +78,8 @@ class UrlUtilsTest(unittest.TestCase): 'product.html?id=200') self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name']), 'product.html?id=200&name=wired') + self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired#id20", ['id', 'foo']), + 'product.html?id=200&foo=bar') def test_canonicalize_url(self): # no query arguments diff --git a/scrapy/trunk/scrapy/utils/url.py b/scrapy/trunk/scrapy/utils/url.py index 091827760..131875a54 100644 --- a/scrapy/trunk/scrapy/utils/url.py +++ b/scrapy/trunk/scrapy/utils/url.py @@ -87,6 +87,7 @@ def url_query_parameter(url, parameter, default=None, keep_blank_values=0): def url_query_cleaner(url, parameterlist=(), sep='&', kvsep='='): """Clean url arguments leaving only those passed in the parameterlist""" try: + url = urlparse.urldefrag(url)[0] base, query = url.split('?', 1) parameters = [pair.split(kvsep, 1) for pair in query.split(sep)] except: