diff --git a/scrapy/trunk/scrapy/contrib/adaptors.py b/scrapy/trunk/scrapy/contrib/adaptors.py index 6a19656b1..565b93728 100644 --- a/scrapy/trunk/scrapy/contrib/adaptors.py +++ b/scrapy/trunk/scrapy/contrib/adaptors.py @@ -3,6 +3,8 @@ Generic Adaptors for item adaptation pipe. Performs tasks usually needed by most cases """ +import re + from scrapy.item.models import BaseAdaptor from scrapy.utils.python import flatten, unique from scrapy.xpath import XPathSelector @@ -12,18 +14,10 @@ from scrapy.conf import settings class ExtendedAdaptor(BaseAdaptor): def do(self, function, a, *args, **kwargs): - """Execute a function of the pipeline examining kwargs for possible - pre/post functions. Also prints a lot of useful debugging info.""" + """Executes an adaptor printing a lot of useful debugging info.""" debug = kwargs.get('debug') fname = function.__name__ - f = kwargs.get('pre_%s' % fname) - if f and a: - if debug: - print " pre_%s | input >" % fname, a - a = f(a) - if debug: - print " pre_%s | output >" % fname, a if debug: print repr(kwargs) @@ -32,21 +26,14 @@ class ExtendedAdaptor(BaseAdaptor): if debug: print " %07s | output>" % fname, a - f = kwargs.get('post_%s' % fname) - if f and a: - if debug: - print " post_%s | input >" % fname, a - a = f(a) - if debug: - print " post_%s | output >" % fname, a return a class ExtractAdaptor(ExtendedAdaptor): - def function(self, item, attrname, location, **pipeargs): - return self.do(self._extract, location, **pipeargs) + def function(self, item, location, **pipeargs): + return self.do(self.extract, location, **pipeargs) - def _extract(self, location, **kwargs): + def extract(self, location, **kwargs): """Extract a list of strings from the location passed. Receives a list of XPathSelectors or an XPathSelector, or a list of strings or a string. @@ -58,7 +45,7 @@ class ExtractAdaptor(ExtendedAdaptor): if not location: return [] if isinstance(location, (list, tuple)): - strings = flatten([self._extract(o, **kwargs) for o in location]) + strings = flatten([self.extract(o, **kwargs) for o in location]) if kwargs.get('remove_dupes', False): strings = unique(strings) if kwargs.get('first', False): @@ -79,3 +66,59 @@ class ExtractAdaptor(ExtendedAdaptor): return strings + +_clean_spaces_re = re.compile("\s+", re.U) +_remove_root_re = re.compile(r'^\s*<.*?>(.*)\s*$', re.DOTALL) +_xml_remove_tags_re = re.compile(r'<[a-zA-Z\/!][^>]*?>') +_xml_remove_cdata_re = re.compile(')', re.S) + +class HtmlCleanAdaptor(ExtendedAdaptor): + + def function(self, item, location, **pipeargs): + return self.do(self.clean, string, **pipeargs) + + def clean(self, string, **kwargs): + """Clean (list of) strings removing newlines, spaces, etc""" + if isinstance(string, list): + return [self.clean(s, **kwargs) for s in string] + + xml = self._remove_tags(string, **kwargs) + + if kwargs.get('remove_root', True) and not kwargs.get('remove_tags', True): + m = _remove_root_re.search(xml) + if m: + xml = m.group(1) + + if kwargs.get('remove_spaces', True): + xml = _clean_spaces_re.sub(' ', xml) + if kwargs.get('strip', True): + xml = xml.strip() + + return xml + + def _remove_tags(self, xml, **kwargs): + if kwargs.get('remove_tags', True): + xml = _xml_remove_tags_re.sub(' ', xml) + return xml + +class XmlCleanAdaptor(HtmlCleanAdaptor): + + def _remove_tags(self, xml, **kwargs): + #process in pieces the text that contains CDATA. The first check is to avoid unnecesary regex check + if _xml_remove_cdata_re.search(xml): + pieces = [] + + for piece in _xml_cdata_split_re.split(xml): + + m = _xml_remove_cdata_re.search(piece) + if m: + if kwargs.get('remove_cdata', True):#remove cdata special tag + pieces.append(HtmlCleanAdaptor._remove_tags(self, m.groups()[0], **kwargs)) + else: + pieces.append(piece)#conserve intact the cdata + else: + pieces.append(HtmlCleanAdaptor._remove_tags(self, piece, **kwargs)) + + xml = "".join(pieces) + return xml diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py index 6f180a123..0f217b8a6 100644 --- a/scrapy/trunk/scrapy/item/models.py +++ b/scrapy/trunk/scrapy/item/models.py @@ -1,19 +1,14 @@ class BaseAdaptor(object): - def function(self, item, attrname, value, **pipeargs): + def function(self, item, value, **pipeargs): raise NotImplemented #default adaptors class ExtractAdaptor(BaseAdaptor): - def function(self, item, attrname, value, **pipeargs): + def function(self, item, value, **pipeargs): if hasattr(value, 'extract'): value = value.extract() return value -class AssignAdaptor(BaseAdaptor): - def function(self, item, attrname, value, **pipeargs): - if not hasattr(item, attrname): - setattr(item, attrname, value) - class ScrapedItem(object): """ This is the base class for all scraped items. @@ -22,7 +17,7 @@ class ScrapedItem(object): * guid (unique global indentifier) * url (URL where that item was scraped from) """ - adaptors_pipe = [ExtractAdaptor(), AssignAdaptor()] + adaptors_pipe = [ExtractAdaptor()] def set_adaptors_pipe(adaptors_pipes): ScrapedItem.adaptors_pipes = adaptors_pipes @@ -30,4 +25,6 @@ class ScrapedItem(object): def attribute(self, name, value, **pipeargs): for adaptor in ScrapedItem.adaptors_pipe: - value = adaptor.function(self, name, value, **pipeargs) + value = adaptor.function(self, value, **pipeargs) + if not hasattr(item, name): + setattr(item, name, value) \ No newline at end of file