From c73dc5ad6c96f7f22c10c22f6e04375915848c02 Mon Sep 17 00:00:00 2001 From: elpolilla Date: Wed, 29 Oct 2008 01:28:04 +0000 Subject: [PATCH] Added normalize_urls adaptor, which was mentioned in the previous changeset, but not actually commited --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40349 --- scrapy/trunk/scrapy/contrib/adaptors/misc.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/trunk/scrapy/contrib/adaptors/misc.py b/scrapy/trunk/scrapy/contrib/adaptors/misc.py index a93ad9f87..06bd16d61 100644 --- a/scrapy/trunk/scrapy/contrib/adaptors/misc.py +++ b/scrapy/trunk/scrapy/contrib/adaptors/misc.py @@ -1,5 +1,6 @@ import re from scrapy.xpath.selector import XPathSelector, XPathSelectorList +from scrapy.utils.url import canonicalize_url from scrapy.utils.misc import extract_regex from scrapy.utils.python import flatten @@ -26,6 +27,13 @@ def strip_list(value): def drop_empty(value): return [ v for v in value if v ] +def normalize_urls(value): + if hasattr(value, '__iter__'): + return [canonicalize_url(url) for url in value] + elif isinstance(value, basestring): + return canonicalize_url(value) + return '' + class Delist(object): """ Input: iterable with strings