diff --git a/scrapy/trunk/scrapy/contrib/adaptors.py b/scrapy/trunk/scrapy/contrib/adaptors.py new file mode 100644 index 000000000..2de56bdee --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/adaptors.py @@ -0,0 +1,79 @@ +""" +Adaptors for item adaptation pipe. +""" + +from scrapy.item.models import BaseAdaptor +from scrapy.utils.python import flatten, unique +from scrapy.xpath import XPathSelector +from scrapy.utils.misc import unquote_html, location_str + +class ExtendedAdaptor(BaseAdaptor): + + def do(self, function, a, *args, **kwargs): + """Execute a function of the pipeline examining kwargs for possible + pre/post functions. Also prints a lot of useful debugging info.""" + + debug = kwargs.get('debug') + fname = function.__name__ + f = kwargs.get('pre_%s' % fname) + if f and a: + if debug: + print " pre_%s | input >" % fname, a + a = f(a) + if debug: + print " pre_%s | output >" % fname, a + + if debug: + print repr(kwargs) + print " %07s | input >" % fname, location_str(a) + a = function(a, *args, **kwargs) + if debug: + print " %07s | output>" % fname, a + + f = kwargs.get('post_%s' % fname) + if f and a: + if debug: + print " post_%s | input >" % fname, a + a = f(a) + if debug: + print " post_%s | output >" % fname, a + return a + +class ExtractAdaptor(ExtendedAdaptor): + + def function(self, item, attrname, location, **pipeargs): + return self.do(self._extract, location, **pipeargs) + + def _extract(self, location, **kwargs): + """Extract a list of strings from the location passed. + Receives a list of XPathSelectors or an XPathSelector, + or a list of strings or a string. + + Return a list of strings extracted. + + This function *always* returns a list. + """ + if not location: + return [] + if isinstance(location, (list, tuple)): + strings = flatten([self._extract(o, **kwargs) for o in location]) + if kwargs.get('remove_dupes', False): + strings = unique(strings) + if kwargs.get('first', False): + return strings[:1] + else: + return strings + # XPathSelector + elif isinstance(location, XPathSelector): + strings = [location.extract()] + # Strings + elif isinstance(location, unicode): + strings = [unquote_html(location, keep_reserved=True)] + elif isinstance(location, str): + encoding = kwargs.get("encoding", settings.get('DEFAULT_DATA_ENCODING')) + strings = [unquote_html(unicode(location, encoding), keep_reserved=True)] + else: + raise TypeError, "unsupported location type: %s" % type(location) + + return strings + diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py index ea99cc43a..0109fe9ee 100644 --- a/scrapy/trunk/scrapy/item/models.py +++ b/scrapy/trunk/scrapy/item/models.py @@ -1,3 +1,19 @@ +class BaseAdaptor: + def function(self, item, attrname, value, **pipeargs): + raise NotImplemented + +#default adaptors +class ExtractAdaptor(BaseAdaptor): + def function(self, item, attrname, value, **pipeargs): + if hasattr(value, 'extract'): + value = value.extract() + return value + +class AssignAdaptor(BaseAdaptor): + def function(self, item, attrname, value, **pipeargs): + if not hasattr(item, attrname): + setattr(item, attrname, value) + class ScrapedItem(object): """ This is the base class for all scraped items. @@ -6,11 +22,12 @@ class ScrapedItem(object): * guid (unique global indentifier) * url (URL where that item was scraped from) """ + adaptors_pipe = [ExtractAdaptor(), AssignAdaptor()] + + def set_adaptors_pipe(adaptors_pipes): + ScrapedItem.adaptors_pipes = adaptors_pipes - def assign(self, name, value): - """Assign an attribute. Can receive a Selector in value which will - extract its data """ + def attribute(self, name, value, **pipeargs): - if hasattr(value, 'extract'): - value = value.extract() - setattr(self, name, value) + for adaptor in ScrapedItem.adaptors_pipe: + value = adaptor.function(self, name, value, **pipeargs) diff --git a/scrapy/trunk/scrapy/templates/settings.tmpl b/scrapy/trunk/scrapy/templates/settings.tmpl index 9a9576da6..2b369cadf 100644 --- a/scrapy/trunk/scrapy/templates/settings.tmpl +++ b/scrapy/trunk/scrapy/templates/settings.tmpl @@ -7,6 +7,8 @@ import __project_name__ BOT_NAME = 'scrapy-bot' BOT_VERSION = '1.0' +DEFAULT_DATA_ENCODING = 'ISO-8859-1' + COMMANDS_MODULE = '__project_name__.commands' COMMANDS_SETTINGS_MODULE = '__project_name__.conf.commands' SPIDER_MODULES = ['__project_name__.spiders'] diff --git a/scrapy/trunk/scrapy/utils/misc.py b/scrapy/trunk/scrapy/utils/misc.py index 9c242938d..0d1fe40cb 100644 --- a/scrapy/trunk/scrapy/utils/misc.py +++ b/scrapy/trunk/scrapy/utils/misc.py @@ -202,4 +202,12 @@ def extract_regex(regex, text, encoding): else: return [unquote_html(unicode(s, encoding), keep_reserved=True) for s in strings] +_regex_type = type(re.compile("", 0)) +def location_str(location): + """Return a human friendly representation of a parser location""" + + if isinstance(location, _regex_type): + return "(regex) " + location.pattern + else: + return location