diff --git a/scrapy/trunk/scrapy/item/adaptors.py b/scrapy/trunk/scrapy/item/adaptors.py
index 17cd58644..67e3243e4 100644
--- a/scrapy/trunk/scrapy/item/adaptors.py
+++ b/scrapy/trunk/scrapy/item/adaptors.py
@@ -1,197 +1,268 @@
import re
+import cPickle as pickle
from traceback import format_exc
+from urlparse import urlparse
from scrapy.xpath.selector import XPathSelector, XPathSelectorList
from scrapy.utils.python import unique, flatten
from scrapy.utils.markup import replace_tags, remove_entities
from scrapy.utils.misc import extract_regex
+from scrapy.conf import settings
+from scrapy import log
+
+class AdaptorPipe(object):
+ """
+ Class that represents an item's attribute pipeline.
+
+ This class contains a dictionary of attributes, matched with a list
+ of adaptors to be run for filtering the input before storing.
+ """
-class AdaptorPipe:
def __init__(self, adaptors_pipe=None):
"""
Receives a dictionary that maps attribute_name to a list of adaptor functions
"""
self.pipes = adaptors_pipe or {}
- def append_adaptor(self, attrname, adaptor):
+ def set_adaptors(self, attr, adaptors):
"""
- Add an adaptor at the end of the provided attribute's pipeline
+ Set the adaptor pipeline that will be used for the specified attribute
"""
- if callable(adaptor):
- if self.pipes.get(attrname):
- self.pipes[attrname].append(adaptor)
- else:
- self.pipes[attrname] = [adaptor]
-
- def execute(self, attrname, value, debug=False):
+ self.pipes[attr] = adaptors
+
+ def execute(self, attrname, value, kwargs):
"""
Execute pipeline for attribute name "attrname" and value "value".
"""
- for function in self.pipes.get(attrname, []):
+ debug = kwargs.get('debug') or all([settings.getbool('LOG_ENABLED'), settings.get('LOGLEVEL') == 'TRACE'])
+
+ for adaptor in self.pipes.get(attrname, []):
try:
if debug:
- print " %07s | input >" % function.func_name, repr(value)
- value = function(value)
+ print " %07s | input >" % adaptor.__name__, repr(value)
+ value = adaptor(kwargs)(value)
if debug:
- print " %07s | output>" % function.func_name, repr(value)
-
+ print " %07s | output >" % adaptor.__name__, repr(value)
+
except Exception, e:
- print "Error in '%s' adaptor. Traceback text:" % function.func_name
+ print "Error in '%s' adaptor. Traceback text:" % adaptor.__name__
print format_exc()
return
-
+
return value
+class AdaptorFunc(object):
+ """
+ This is the base class for adaptors.
+
+ An adaptor is just an object subclassed from this class
+ which defines the __call__ method, and receives/returns only
+ one value.
+
+ You can send the adaptor some extra options while creating it (just before running it)
+ through **kwargs, and managing them by overriding the __init__ method, as shown on
+ the UnquoteAdaptor, for example.
+ """
+ def __init__(self, kwargs={}):
+ pass
+
+ def __call__(self):
+ raise NotImplementedError('You must define the __call__ method to create and use an adaptor')
############
# Adaptors #
############
-def extract(location):
+class ExtractAdaptor(AdaptorFunc):
"""
This adaptor extracts a list of strings
from 'location', which can be either a list (or tuple),
or an XPathSelector.
- This function *always* returns a list.
+ This adaptor *always* returns a list.
"""
- if not location:
- return []
- elif isinstance(location, XPathSelectorList):
- return flatten([extract(o) for o in location])
- elif isinstance(location, XPathSelector):
- return location.extract()
- elif isinstance(location, (list, tuple)):
- return flatten(location)
- elif isinstance(location, basestring):
- return [location]
-def _absolutize_links(rel_links, current_url, base_url):
- abs_links = []
- for link in rel_links:
- if link.startswith('/'):
- abs_links.append('%s%s' % (base_url, link))
- elif link.startswith('http://'):
- abs_links.append(link)
- else:
- abs_links.append('%s/%s' % (current_url, link))
- return abs_links
-
-def extract_links(locations):
+ def __call__(self, location):
+ if not location:
+ return []
+ elif isinstance(location, (XPathSelector, XPathSelectorList)):
+ return flatten(location.extract())
+ elif isinstance(location, (list, tuple)):
+ return flatten(map(lambda x: x.extract() if isinstance(x, (XPathSelector, XPathSelectorList)) else x, flatten(location)))
+ elif isinstance(location, basestring):
+ return [location]
+
+class ExtractImagesAdaptor(AdaptorFunc):
"""
This adaptor receives either an XPathSelector containing
the desired locations for finding urls, or a tuple like (xpath, regexp)
containing the xpath locations to look in, and a regular expression
to parse those locations.
- In any case, this adaptor returns a list of absolute urls extracted.
+ In any case, this adaptor returns a list containing the absolute urls extracted.
"""
- ret = []
- if locations:
- regexp = None
- if isinstance(locations, XPathSelector):
- locations = XPathSelectorList([locations])
- elif isinstance(locations, tuple):
- locations, regexp = locations
-
- if isinstance(locations, XPathSelectorList):
- if regexp:
- ret = locations.re(regexp)
- else:
- for selector in locations:
- if selector.xmlNode.type == 'element':
- if selector.xmlNode.name == 'a':
- children = selector.x('child::*')
- if len(children) > 1:
- ret.extend(selector.x('.//@href'))
- ret.extend(selector.x('.//@src'))
- elif len(children) == 1 and children[0].xmlNode.name == 'img':
- ret.extend(children.x('@src'))
- else:
- ret.extend(selector.x('@href'))
- elif selector.xmlNode.name == 'img':
- ret.extend(selector.x('@src'))
- else:
- ret.extend(selector.x('.//@href'))
- ret.extend(selector.x('.//@src'))
- elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']:
- ret.append(selector)
- ret = [selector.extract() for selector in ret]
- current_url, base_url = re.search(r'((http://(?:www\.)?[\w\d\.-]+?)(?:/|$).*)', locations[0].response.url).groups()
- ret = _absolutize_links(ret, current_url, base_url)
- return ret
-def to_unicode(value):
+ def __init__(self, kwargs):
+ self.base_url = kwargs.get('base_url')
+ self.response = kwargs.get('response')
+ if not self.response and not self.base_url:
+ raise AttributeError('You must specify either a response or a base_url to the ExtractImages adaptor.')
+
+ def extract_from_xpath(self, selector):
+ ret = []
+
+ if selector.xmlNode.type == 'element':
+ if selector.xmlNode.name == 'a':
+ children = selector.x('child::*')
+ if len(children) > 1:
+ ret.extend(selector.x('.//@href'))
+ ret.extend(selector.x('.//@src'))
+ elif len(children) == 1 and children[0].xmlNode.name == 'img':
+ ret.extend(children.x('@src'))
+ else:
+ ret.extend(selector.x('@href'))
+ elif selector.xmlNode.name == 'img':
+ ret.extend(selector.x('@src'))
+ else:
+ ret.extend(selector.x('.//@href'))
+ ret.extend(selector.x('.//@src'))
+ elif selector.xmlNode.type == 'attribute' and selector.xmlNode.name in ['href', 'src']:
+ ret.append(selector)
+
+ return ret
+
+ def absolutize_link(self, base_url, link):
+ base_url = urlparse(base_url)
+ ret = []
+
+ if link.startswith('/'):
+ ret.append('http://%s%s' % (base_url.hostname, link))
+ elif link.startswith('http://'):
+ ret.append(link)
+ else:
+ ret.append('http://%s%s/%s' % (base_url.hostname, base_url.path, link))
+
+ return ret
+
+ def __call__(self, locations):
+ rel_links = []
+ for location in flatten(locations):
+ if isinstance(location, (XPathSelector, XPathSelectorList)):
+ rel_links.extend(self.extract_from_xpath(location))
+ else:
+ rel_links.append(location)
+ rel_links = ExtractAdaptor()(rel_links)
+
+ if self.response:
+ return flatten([self.absolutize_link(self.response.url, link) for link in rel_links])
+ elif self.base_url:
+ return flatten([self.absolutize_link(self.base_url, link) for link in rel_links])
+ else:
+ abs_links = []
+ for link in rel_links:
+ if link.startswith('http://'):
+ abs_links.append(link)
+ else:
+ log.msg('Couldnt get the absolute url for "%s". Ignoring link...' % link, 'WARNING')
+ return abs_links
+
+class BoolAdaptor(AdaptorFunc):
+ def __call__(self, value):
+ return bool(value)
+
+class ToUnicodeAdaptor(AdaptorFunc):
"""
Receives a list of strings, converts
it to unicode, and returns a new list.
"""
- if isinstance(value, (list, tuple)):
- return [ unicode(v) for v in value ]
- else:
- raise TypeError('to_unicode adaptor must receive a list or a tuple.')
-def regex(expr):
- """
- This factory function returns a ready-to-use
- adaptor for the specified regular expression.
- This adaptor will accept either an XPathSelectorList
- or a list of strings, and will apply the provided regular
- expression to each of its members.
+ def __call__(self, value):
+ if isinstance(value, (list, tuple)):
+ return [ unicode(v) for v in value ]
+ else:
+ raise TypeError('ToUnicodeAdaptor must receive either a list or a tuple.')
- This adaptor always returns a list of strings.
+class RegexAdaptor(AdaptorFunc):
"""
- def _regex(value):
- if isinstance(value, (XPathSelector, XPathSelectorList)):
- return value.re(expr)
- elif isinstance(value, list) and value:
- return flatten([extract_regex(expr, string, 'utf-8') for string in value])
- return value
- return _regex
+ This adaptor must receive either a list of strings or an XPathSelector
+ and return a new list with the matches of the given strings with the given regular
+ expression (which is passed by a keyword argument, and is mandatory for this adaptor).
+ """
+
+ def __init__(self, kwargs):
+ self.regex = kwargs.get('regex')
-def unquote_all(value):
+ def __call__(self, value):
+ if self.regex:
+ if isinstance(value, (XPathSelector, XPathSelectorList)):
+ return value.re(self.regex)
+ elif isinstance(value, list) and value:
+ return flatten([extract_regex(self.regex, string, 'utf-8') for string in value])
+ return value
+
+class UnquoteAdaptor(AdaptorFunc):
"""
Receives a list of strings, removes all of the
entities the strings may have, and returns
a new list
"""
- return [ remove_entities(v) for v in value ]
-def unquote(value):
- """
- Receives a list of strings, removes all of the entities
- the strings may have (except for < and &), and
- returns a new list
- """
- return [ remove_entities(v, keep=['lt', 'amp']) for v in value ]
+ def __init__(self, kwargs={}):
+ self.keep = kwargs.get('keep', ['lt', 'amp'])
-def remove_tags(value):
- return [ replace_tags(v) for v in value ]
+ def __call__(self, value):
+ return [ remove_entities(v, keep=self.keep) for v in value ]
+
+class RemoveTagsAdaptor(AdaptorFunc):
+ def __call__(self, value):
+ return [ replace_tags(v) for v in value ]
-_remove_root_re = re.compile(r'^\s*<.*?>(.*)\s*$', re.DOTALL)
-def _remove_root(value):
- m = _remove_root_re.search(value)
- if m:
- value = m.group(1)
- return value
+class RemoveRootAdaptor(AdaptorFunc):
+ _remove_root_re = re.compile(r'^\s*<.*?>(.*)\s*$', re.DOTALL)
+ def _remove_root(self, value):
+ m = self._remove_root_re.search(value)
+ if m:
+ value = m.group(1)
+ return value
-def remove_root(value):
- return [ _remove_root(v) for v in value ]
+ def __call__(self, value):
+ return [ self._remove_root(v) for v in value ]
-_clean_spaces_re = re.compile("\s+", re.U)
-def remove_multispaces(value):
- return [ _clean_spaces_re.sub(' ', v) for v in value ]
+class CleanSpacesAdaptor(AdaptorFunc):
+ _clean_spaces_re = re.compile("\s+", re.U)
+ def __call__(self, value):
+ return [ self._clean_spaces_re.sub(' ', v) for v in value ]
-def strip(value):
- return [ v.strip() for v in value ]
+class StripAdaptor(AdaptorFunc):
+ def __call__(self, value):
+ return [ v.strip() for v in value ]
-def drop_empty_elements(value):
- return [ v for v in value if v ]
+class DropEmptyAdaptor(AdaptorFunc):
+ def __call__(self, value):
+ return [ v for v in value if v ]
-def delist(value):
- return ' '.join(value)
+class DelistAdaptor(AdaptorFunc):
+ def __init__(self, kwargs={}):
+ self.delimiter = kwargs.get('join_delimiter', ' ')
+
+ def __call__(self, value):
+ return self.delimiter.join(value)
+class PickleAdaptor(AdaptorFunc):
+ def __call__(self, value):
+ return pickle.dumps(value)
+class DePickleAdaptor(AdaptorFunc):
+ def __call__(self, value):
+ return pickle.loads(value)
+
+class UniqueAdaptor(AdaptorFunc):
+ def __call__(self, value):
+ return unique(value)
+
+class FlattenAdaptor(AdaptorFunc):
+ def __call__(self, value):
+ return flatten(value)
#############
# Pipelines #
@@ -200,42 +271,38 @@ def delist(value):
The following methods automatically generate adaptor pipelines
for some basic datatypes, according to the parameters you pass them.
"""
-def single_pipeline(remove_root=True, remove_tags=True, do_unquote=True):
- pipe = [ extract,
- unique,
- to_unicode,
- drop_empty_elements,
- remove_multispaces,
- strip,
+def single_pipeline(do_remove_root=True, do_remove_tags=True, do_unquote=True):
+ pipe = [ ExtractAdaptor,
+ UniqueAdaptor,
+ ToUnicodeAdaptor,
+ DropEmptyAdaptor,
+ CleanSpacesAdaptor,
+ StripAdaptor,
]
- if remove_root:
- pipe.insert(4, remove_root)
- if remove_tags:
- pipe.insert(5, remove_tags)
+
+ if do_remove_root:
+ pipe.insert(4, RemoveRootAdaptor)
+ if do_remove_tags:
+ pipe.insert(5, RemoveTagsAdaptor)
if do_unquote:
- pipe.append(unquote)
- return pipe + [delist]
+ pipe.append(UnquoteAdaptor)
+ return pipe + [DelistAdaptor]
def url_pipeline():
- return [ extract_links,
- unique,
- to_unicode,
- drop_empty_elements,
+ return [ ExtractImagesAdaptor,
+ UniqueAdaptor,
+ ToUnicodeAdaptor,
+ DropEmptyAdaptor,
]
-def list_pipeline(do_extract=True):
- pipe = []
- if do_extract:
- pipe.append(extract)
- return pipe + [ unique,
- to_unicode,
- drop_empty_elements,
- unquote,
- remove_tags,
- remove_root,
- strip,
- ]
+list_pipeline = [ ExtractAdaptor,
+ UniqueAdaptor,
+ ToUnicodeAdaptor,
+ DropEmptyAdaptor,
+ UnquoteAdaptor,
+ RemoveTagsAdaptor,
+ RemoveRootAdaptor,
+ StripAdaptor,
+ ]
-def list_join_pipeline(delimiter='\t'):
- return list_pipeline() + [delimiter.join]
-
+list_join_pipeline = list_pipeline + [DelistAdaptor]
diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py
index a68568104..7157dc761 100644
--- a/scrapy/trunk/scrapy/item/models.py
+++ b/scrapy/trunk/scrapy/item/models.py
@@ -1,4 +1,5 @@
from scrapy.item.adaptors import AdaptorPipe
+from scrapy.conf import settings
class ScrapedItem(object):
"""
@@ -7,18 +8,31 @@ class ScrapedItem(object):
that identifies uniquely the given scraped item.
"""
- def setadaptors(self, adaptors_pipe):
+ def __init__(self, adaptors_pipe={}):
+ #self.set_adaptors(adaptors_pipe)
+ pass
+
+ def set_adaptors(self, adaptors_pipe):
"""
- Set adaptors to use for this item. Receives a dict of adaptors and
- returns the item itself.
+ Set the adaptors to use for this item. Receives a dict of the adaptors
+ desired for each attribute and returns the item itself.
"""
- object.__setattr__(self, '_adaptors_pipe', AdaptorPipe(adaptors_pipe))
+ setattr(self, '_adaptors_pipe', AdaptorPipe(adaptors_pipe))
return self
- def append_adaptor(self, attrname, adaptor):
- self._adaptors_pipe.append_adaptor(attrname, adaptor)
-
- def attribute(self, attrname, value):
- value = self._adaptors_pipe.execute(attrname, value)
- if not hasattr(self, attrname) or not getattr(self, attrname):
- setattr(self, attrname, value)
+ def set_attrib_adaptors(self, attrib, adaptors):
+ """
+ Set the adaptors (from a list or tuple) to be used for a specific attribute.
+ """
+ self._adaptors_pipe.set_adaptors(attrib, adaptors)
+
+ def attribute(self, attrname, value, **kwargs):
+ val = self._adaptors_pipe.execute(attrname, value, kwargs)
+ if not getattr(self, attrname, None) or kwargs.get('override'):
+ setattr(self, attrname, val)
+
+ def __sub__(self, other):
+ raise NotImplementedError
+
+class ItemDelta(object):
+ pass
diff --git a/scrapy/trunk/scrapy/tests/test_adaptors.py b/scrapy/trunk/scrapy/tests/test_adaptors.py
index df1627b5a..c8e1db286 100644
--- a/scrapy/trunk/scrapy/tests/test_adaptors.py
+++ b/scrapy/trunk/scrapy/tests/test_adaptors.py
@@ -7,17 +7,17 @@ from scrapy.http import Response, ResponseBody
class AdaptorsTestCase(unittest.TestCase):
def test_extract(self):
sample_xsel = XmlXPathSelector(text='