RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactored extract adaptor 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40671
This commit is contained in:
elpolilla 2009-01-07 12:48:15 +00:00
parent 66852b8291
commit cd148c946e
3 changed files with 24 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/trunk/scrapy/contrib/adaptors/__init__.py
View File

@ -1,4 +1,4 @@
from scrapy.contrib.adaptors.extraction import extract, extract_unquoted, ExtractImages
from scrapy.contrib.adaptors.extraction import extract, ExtractImages
from scrapy.contrib.adaptors.markup import remove_tags, remove_root, Unquote
from scrapy.contrib.adaptors.misc import to_unicode, clean_spaces, strip, drop_empty, canonicalize_urls, Delist, Regex
from scrapy.utils.python import unique, flatten

49
scrapy/trunk/scrapy/contrib/adaptors/extraction.py
View File

@ -10,39 +10,32 @@ from scrapy.utils.url import is_url
from scrapy.utils.python import flatten
from scrapy.xpath.selector import XPathSelector, XPathSelectorList
def _extract(locations, extractor='extract'):
if isinstance(locations, (XPathSelector, XPathSelectorList)):
return flatten(getattr(locations, extractor)())
elif hasattr(locations, '__iter__'):
return flatten([getattr(x, extractor)() if isinstance(x, (XPathSelector, XPathSelectorList)) else x for x in flatten(locations)])
elif isinstance(locations, basestring):
return [locations]
else:
return []
def extract(locations):
def extract(locations, use_unquote=True):
"""
This adaptor extracts a list of strings
from 'locations', which can be either an iterable,
or an XPathSelector/XPathSelectorList.
This adaptor tries to extract data from the given locations.
Any XPathSelector in it will be extracted, and any other data
will be added as-is to the result.
Input: XPathSelector, XPathSelectorList, iterable, basestring
Output: list of unicodes
"""
return _extract(locations)
If an XPathSelector is a text/cdata node, and `use_unquote`
is True, that selector will be extracted using the `extract_unquoted`
method; otherwise, the `extract` method will be used.
def extract_unquoted(locations):
Input: anything
Output: list of extracted selectors plus anything else in the input
"""
This adaptor extracts a list of unquoted strings
from 'locations', which can be either an iterable,
or an XPathSelector/XPathSelectorList.
The difference between this and the extract adaptor is
that this adaptor will only extract text nodes and unquote them.
Input: XPathSelector, XPathSelectorList, iterable, basestring
Output: list of unicodes
"""
return _extract(locations, 'extract_unquoted')
locations = flatten([locations])
result = []
for location in locations:
if isinstance(location, XPathSelector):
if location.xmlNode.type in ('text', 'cdata') and use_unquote:
result.append(location.extract_unquoted())
else:
result.append(location.extract())
else:
result.append(location)
return result
class ExtractImages(object):
"""

4
scrapy/trunk/scrapy/tests/test_adaptors.py
View File

@ -75,8 +75,8 @@ class AdaptorsTestCase(unittest.TestCase):
def test_extract_unquoted(self):
x = self.get_selector('example.com', 'http://www.example.com/test_unquoted', 'extr_unquoted.xml', selector=XmlXPathSelector)
self.assertEqual(adaptors.extract_unquoted(x.x('//tag1/text()')), [u'test text & &', u'more test text & >', u'blah&blah'])
self.assertEqual(adaptors.extract_unquoted(x.x('//tag2/text()')), [u'blaheawfds<'])
self.assertEqual(adaptors.extract(x.x('//tag1/text()')), [u'test text & &', u'more test text &amp; &gt;', u'blah&blah'])
self.assertEqual(adaptors.extract(x.x('//tag2/text()')), [u'blaheawfds<'])
def test_extract_links(self):
test_data = """<html><body>