diff --git a/scrapy/contrib_exp/adaptors/__init__.py b/scrapy/contrib_exp/adaptors/__init__.py deleted file mode 100644 index 2b6481183..000000000 --- a/scrapy/contrib_exp/adaptors/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from scrapy.contrib_exp.adaptors.extraction import extract, ExtractImageLinks -from scrapy.contrib_exp.adaptors.markup import remove_tags, remove_root, replace_escape, unquote -from scrapy.contrib_exp.adaptors.misc import to_unicode, clean_spaces, strip, drop_empty, delist, Regex -from scrapy.contrib_exp.adaptors.date import to_date - diff --git a/scrapy/contrib_exp/adaptors/date.py b/scrapy/contrib_exp/adaptors/date.py deleted file mode 100644 index 989f14c0b..000000000 --- a/scrapy/contrib_exp/adaptors/date.py +++ /dev/null @@ -1,14 +0,0 @@ -import time - -def to_date(format): - """ - Converts a date string to a YYYY-MM-DD one suitable for DateField - - format is the format string passed to time.strptime - - Input: string/unicode - Output: string - """ - def _to_date(value): - return time.strftime('%Y-%m-%d', time.strptime(value, format)) - return _to_date diff --git a/scrapy/contrib_exp/adaptors/extraction.py b/scrapy/contrib_exp/adaptors/extraction.py deleted file mode 100644 index 2d71b70a1..000000000 --- a/scrapy/contrib_exp/adaptors/extraction.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -Adaptors related with extraction of data -""" - -from scrapy.utils.url import urljoin_rfc -from scrapy.utils.response import get_base_url -from scrapy.utils.python import unicode_to_str -from scrapy.xpath.selector import XPathSelector, XPathSelectorList -from scrapy.contrib.linkextractors.image import HTMLImageLinkExtractor - -def extract(location, adaptor_args=None): - """ - This adaptor tries to extract data from the given locations. - Any XPathSelector in it will be extracted, and any other data - will be added as-is to the result. - - If an XPathSelector is a text/cdata node, and `use_unquote` - is True, that selector will be extracted using the `extract_unquoted` - method; otherwise, the `extract` method will be used. - - Input: anything - Output: tuple of extracted selectors plus anything else in the input - """ - use_unquote = adaptor_args.get('use_unquote', True) if adaptor_args else True - - if isinstance(location, XPathSelectorList): - ret = location.extract() - elif isinstance(location, XPathSelector): - if location.xmlNode.type in ('text', 'cdata') and use_unquote: - ret = location.extract_unquoted() - else: - ret = location.extract() - ret = ret if hasattr(ret, '__iter__') else [ret] - elif isinstance(location, list): - ret = tuple(location) - else: - ret = tuple([location]) - - return tuple(x for x in ret if x is not None) - -class ExtractImageLinks(object): - """ - This adaptor may receive either XPathSelectors pointing to - the desired locations for finding image urls, or just a list of - XPath expressions (which will be turned into selectors anyway). - - Input: XPathSelector, XPathSelectorList, iterable - Output: tuple of urls (strings) - """ - def __init__(self, response=None, canonicalize=True): - self.response = response - self.canonicalize = canonicalize - - def __call__(self, locations): - if not locations: - return tuple() - elif not hasattr(locations, '__iter__'): - locations = [locations] - - selectors, raw_links = [], [] - for location in locations: - if isinstance(location, (XPathSelector, XPathSelectorList)): - selectors.append(location) - else: - raw_links.append(location) - - if raw_links: - base_url = get_base_url(self.response) - raw_links = [urljoin_rfc(base_url, unicode_to_str(rel_url), self.response.encoding) for rel_url in raw_links] - - lx = HTMLImageLinkExtractor(locations=selectors, canonicalize=self.canonicalize) - urls = map(lambda link: link.url, lx.extract_links(self.response)) - return tuple(urls + raw_links) - diff --git a/scrapy/contrib_exp/adaptors/markup.py b/scrapy/contrib_exp/adaptors/markup.py deleted file mode 100644 index d7d4eacec..000000000 --- a/scrapy/contrib_exp/adaptors/markup.py +++ /dev/null @@ -1,64 +0,0 @@ -import re -from scrapy.utils.markup import replace_tags, replace_escape_chars, unquote_markup -from scrapy.utils.python import str_to_unicode - -def remove_tags(tags=()): - """ - Factory that returns an adaptor for removing each - tag in the `tags` parameter found in the given value. - If no `tags` are specified, all of them are removed. - - Input: string/unicode - Output: unicode - """ - def _remove_tags(value): - return replace_tags(value, tags) - return _remove_tags - -_remove_root_re = re.compile(r'^\s*<.*?>(.*)\s*$', re.DOTALL) -def remove_root(value): - """ - This adaptor removes the root tag of the given string/unicode, - if it's found. - - Input: string/unicode - Output: unicode - """ - m = _remove_root_re.search(value) - if m: - value = m.group(1) - return str_to_unicode(value) - -def replace_escape(which_ones=('\n', '\t', '\r'), replace_by=u''): - """ - Factory that returns an adaptor for removing/replacing each escape - character in the `wich_ones` parameter found in the given value. - - If `replace_by` is given, escape characters are replaced by that - text, else they're removed. - - Input: string/unicode - Output: unicode - - """ - def _replace_escape(value): - return replace_escape_chars(value, which_ones, replace_by) - return _replace_escape - -def unquote(keep=None): - """ - This factory returns an adaptor that - receives a string or unicode, removes all of the - CDATAs and entities (except the ones in CDATAs, and the ones - you specify in the `keep` parameter) and then, returns a new - string or unicode. - - Input: string/unicode - Output: string/unicode - """ - default_keep = [] if keep is None else keep - - def unquote(value, adaptor_args): - keep = adaptor_args.get('keep_entities', default_keep) - return unquote_markup(value, keep=keep) - return unquote diff --git a/scrapy/contrib_exp/adaptors/misc.py b/scrapy/contrib_exp/adaptors/misc.py deleted file mode 100644 index c7d9ec6d4..000000000 --- a/scrapy/contrib_exp/adaptors/misc.py +++ /dev/null @@ -1,96 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -from scrapy.xpath.selector import XPathSelector, XPathSelectorList -from scrapy.utils.misc import extract_regex -from scrapy.utils.python import flatten, str_to_unicode, unicode_to_str -from scrapy.contrib.item.adaptors import adaptize - -def to_unicode(value, adaptor_args): - """ - Receives a string and converts it to unicode - using the given encoding (if specified, else utf-8 is used) - and returns a new unicode object. - E.g: - >> to_unicode('it costs 20\xe2\x82\xac, or 30\xc2\xa3') - [u'it costs 20\u20ac, or 30\xa3'] - - Input: string - Output: unicode - """ - if not isinstance(value, basestring): - value = str(value) - return str_to_unicode(value, adaptor_args.get('encoding', 'utf-8')) - -_clean_spaces_re = re.compile("\s+", re.U) -def clean_spaces(value): - """ - Converts multispaces into single spaces for the given string. - E.g: - >> clean_spaces(u'Hello sir') - u'Hello sir' - - Input: string/unicode - Output: unicode - """ - return _clean_spaces_re.sub(' ', str_to_unicode(value)) - -def strip(value): - """ - Removes any spaces at both the start and the ending - of the provided string. - E.g: - >> strip(' hi buddies ') - u'hi buddies' - - Input: string/unicode - Output: string/unicode - """ - return value.strip() - -def drop_empty(value): - """ - Removes any index that evaluates to None - from the provided iterable. - E.g: - >> drop_empty([0, 'this', None, 'is', False, 'an example']) - ['this', 'is', 'an example'] - - Input: iterable - Output: list - """ - return [ v for v in value if v ] - -def delist(delimiter=''): - """ - This factory returns and adaptor that joins - an iterable with the specified delimiter. - - Input: iterable of strings/unicodes - Output: string/unicode - """ - def delist(value, adaptor_args): - delim = adaptor_args.get('join_delimiter', delimiter) - return delim.join(value) - return delist - -class Regex(object): - """ - This adaptor must receive either a list of strings or an XPathSelector - and return a new list with the matches of the given strings with the given regular - expression (which is passed by a keyword argument, and is mandatory for this adaptor). - - Input: XPathSelector, XPathSelectorList, iterable - Output: list of unicodes - """ - def __init__(self, regex=r''): - self.regex = regex - - def __call__(self, value): - if self.regex: - if isinstance(value, (XPathSelector, XPathSelectorList)): - return value.re(self.regex) - elif hasattr(value, '__iter__'): - return flatten([extract_regex(self.regex, string, 'utf-8') for string in value]) - return value - diff --git a/scrapy/contrib_exp/newitem/__init__.py b/scrapy/contrib_exp/newitem/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib_exp/newitem/adaptors.py b/scrapy/contrib_exp/newitem/adaptors.py deleted file mode 100644 index 8d1fb5ef1..000000000 --- a/scrapy/contrib_exp/newitem/adaptors.py +++ /dev/null @@ -1,111 +0,0 @@ -from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import get_func_args - - -def adaptize(func): - func_args = get_func_args(func) - if 'adaptor_args' in func_args: - return func - - def _adaptor(value, adaptor_args): - return func(value) - return _adaptor - - -def IDENTITY(v): - return v - - -class ItemAdaptorMeta(type): - def __new__(meta, class_name, bases, attrs): - da = attrs.get('default_adaptor') - if da: - attrs['default_adaptor'] = staticmethod(adaptize(da)) - - cls = type.__new__(meta, class_name, bases, attrs) - - cls._field_adaptors = cls._field_adaptors.copy() - - if cls.item_class: - for item_field in cls.item_class.fields: - if item_field in attrs: - adaptor = adaptize(attrs[item_field]) - cls._field_adaptors[item_field] = adaptor - setattr(cls, item_field, staticmethod(adaptor)) - return cls - - def __getattr__(cls, name): - if name in cls.item_class.fields: - return cls.default_adaptor - raise AttributeError(name) - - -class ItemAdaptor(object): - __metaclass__ = ItemAdaptorMeta - - item_class = None - default_adaptor = IDENTITY - - _field_adaptors = {} - - def __init__(self, response=None, item=None): - self.item_instance = item if item else self.item_class() - self._response = response - - def __setattr__(self, name, value): - if (name.startswith('_') or name == 'item_instance' \ - or name == 'default_adaptor'): - return object.__setattr__(self, name, value) - - fa = self._field_adaptors.get(name, self.default_adaptor) - - adaptor_args = {'response': self._response, 'item': self.item_instance} - ovalue = fa(value, adaptor_args=adaptor_args) - self.item_instance[name] = ovalue - - def __getattribute__(self, name): - if (name.startswith('_') or name.startswith('item_') \ - or name == 'default_adaptor'): - return object.__getattribute__(self, name) - - return self.item_instance[name] - - -def adaptor(*funcs, **default_adaptor_args): - """A pipe adaptor implementing the tree adaption logic - - It takes multiples unnamed arguments used as functions of the pipe, and - keywords used as adaptor_args to be passed to functions that supports it - - If an adaptor function returns a list of values, each value is used as - input for next adaptor function - - Always returns a list of values - """ - - _funcs = [] - for func in funcs: - accepts_args = 'adaptor_args' in get_func_args(func) - _funcs.append((func, accepts_args)) - - def _adaptor(value, adaptor_args=None): - values = arg_to_iter(value) - aargs = default_adaptor_args - if adaptor_args: - aargs = aargs.copy() - aargs.update(adaptor_args) - pipe_kwargs = {'adaptor_args': aargs} - for func, accepts_args in _funcs: - next = [] - kwargs = pipe_kwargs if accepts_args else {} - for val in values: - val = func(val, **kwargs) - if hasattr(val, '__iter__'): - next.extend(val) - elif val is not None: - next.append(val) - values = next - return list(values) - - return _adaptor - diff --git a/scrapy/tests/test_adaptors.py b/scrapy/tests/test_adaptors.py deleted file mode 100644 index 233d1f530..000000000 --- a/scrapy/tests/test_adaptors.py +++ /dev/null @@ -1,196 +0,0 @@ -# -*- coding: utf8 -*- -import unittest -import re - -from scrapy.contrib.item.adaptors import AdaptorPipe -from scrapy.contrib_exp import adaptors -from scrapy.http import HtmlResponse, Headers -from scrapy.xpath.selector import HtmlXPathSelector, XmlXPathSelector -from scrapy.tests import get_testdata - -class AdaptorPipeTestCase(unittest.TestCase): - def test_pipe_init(self): - self.assertRaises(TypeError, AdaptorPipe, [adaptors.extract, 'a string']) - - def test_adaptor_args(self): - def sample_adaptor(value, adaptor_args): - '''Dummy adaptor that joins the received value with the given string''' - sample_text = adaptor_args.get('sample_arg', 'sample text 1') - return '%s "%s"' % (value, sample_text) - - sample_value = 'hi, this is my text:' - sample_pipe = AdaptorPipe([sample_adaptor]) - self.assertEqual(sample_pipe(sample_value), ('hi, this is my text: "sample text 1"', )) - self.assertEqual(sample_pipe(sample_value, {'sample_arg': 'foobarfoobar'}), - ('hi, this is my text: "foobarfoobar"', )) - - def test_add(self): - pipe1 = AdaptorPipe([adaptors.extract]) - pipe2 = [adaptors.remove_tags] - pipe3 = (adaptors.remove_root, ) - sample_callable = dir - - self.assertTrue(isinstance(pipe1 + pipe1, AdaptorPipe)) - self.assertTrue(isinstance(pipe1 + pipe2, AdaptorPipe)) - self.assertTrue(isinstance(pipe1 + pipe3, AdaptorPipe)) - self.assertTrue(isinstance(pipe1 + sample_callable, AdaptorPipe)) - -class AdaptorsTestCase(unittest.TestCase): - - def get_selector(self, domain, url, sample_filename, headers=None, selector=HtmlXPathSelector): - body = get_testdata('adaptors', sample_filename) - response = HtmlResponse(url=url, headers=Headers(headers), status=200, body=body) - return selector(response) - - def test_extract(self): - def check_extractor(x, pound=True, euro=True): - poundre = re.compile(r'(.*?)') - eurore = re.compile(r'(.*?)') - - if pound: - self.assertEqual(adaptors.extract(x.x("//span[@class='pound']/text()")), (u'\xa3', )) - self.assertEqual(adaptors.extract(x.x("//span[@class='pound']/@value")), (u'\xa3', )) - self.assertEqual(adaptors.extract(x.re(poundre)), (u'\xa3', )) - self.assertEqual(adaptors.extract(x.x("//span[@class='poundent']/text()")), (u'\xa3', )) - self.assertEqual(adaptors.extract(x.x("//span[@class='poundent']/@value")), (u'\xa3', )) - self.assertEqual(adaptors.extract(x.x("//span[@class='poundnum']/text()")), (u'\xa3', )) - self.assertEqual(adaptors.extract(x.x("//span[@class='poundnum']/@value")), (u'\xa3', )) - if euro: - self.assertEqual(adaptors.extract(x.x("//span[@class='euro']/text()")), (u'\u20ac', )) - self.assertEqual(adaptors.extract(x.x("//span[@class='euro']/@value")), (u'\u20ac', )) - self.assertEqual(adaptors.extract(x.re(eurore)), (u'\u20ac', )) - self.assertEqual(adaptors.extract(x.x("//span[@class='euroent']/text()")), (u'\u20ac', )) - self.assertEqual(adaptors.extract(x.x("//span[@class='euroent']/@value")), (u'\u20ac', )) - self.assertEqual(adaptors.extract(x.x("//span[@class='euronum']/text()")), (u'\u20ac', )) - self.assertEqual(adaptors.extract(x.x("//span[@class='euronum']/@value")), (u'\u20ac', )) - - x = self.get_selector('example.com', - 'http://www.example.com/test/utf8', - 'enc-utf8.html', - {}) - check_extractor(x) - - x = self.get_selector('example.com', - 'http://www.example.com/test/latin1', - 'enc-latin1.html', - {}) - check_extractor(x, euro=False) - - x = self.get_selector('example.com', - 'http://www.example.com/test/cp1252', - 'enc-cp1252.html', - {}) - check_extractor(x) - - # HTTP utf-8 | Meta latin1 | Content ascii | using entities - x = self.get_selector('example.com', - 'http://www.example.com/test/ascii', - 'enc-ascii.html', - {'Content-Type': ['text/html; charset=utf-8']}) - check_extractor(x, pound=False, euro=False) - - # Test for inconsistencies between HTTP header encoding and - # META header encoding. It must prefer HTTP header like browsers do - x = self.get_selector('example.com', - 'http://www.example.com/test/utf8-meta-latin1', - 'enc-utf8-meta-latin1.html', - {'Content-Type': ['text/html; charset=utf-8']}) - check_extractor(x) - - - def test_extract_unquoted(self): - x = self.get_selector('example.com', 'http://www.example.com/test_unquoted', 'extr_unquoted.xml', selector=XmlXPathSelector) - - # test unquoting - self.assertEqual(adaptors.extract(x.x('//tag1/text()[2]')[0]), (u'more test text & >', )) - self.assertEqual(adaptors.extract(x.x('//tag2/text()[1]')[0]), (u'blaheawfds<', )) - - # test without unquoting - self.assertEqual(adaptors.extract(x.x('//tag1/text()'), {'use_unquote': False}), - (u'test text & &', u'', u'blah&blah')) - self.assertEqual(adaptors.extract(x.x('//tag2/text()'), {'use_unquote': False}), (u'blaheawfds<', )) - - def test_extract_image_links(self): - test_data = """ -
- lala1 - lala2 - lala3 - - something1 - something2 -
- """ - sample_response = HtmlResponse('http://foobar.com/dummy', body=test_data) - sample_selector = HtmlXPathSelector(sample_response) - sample_adaptor = adaptors.ExtractImageLinks(response=sample_response) - - self.assertEqual(sample_adaptor(None), tuple()) - self.assertEqual(sample_adaptor([]), tuple()) - self.assertEqual(sample_adaptor([sample_selector.x('//@href'), 'http://foobar.com/my_image.gif']), ( - 'http://foobar.com/lala1/lala1.html', - 'http://foobar.com/lala2.html', - 'http://foobar.com/pepepe/papapa/lala3.html', - 'http://foobar.com/lala4.html', - 'http://foobar.com/my_image.gif', - )) - self.assertEqual(sample_adaptor(sample_selector.x('//a')), ( - 'http://foobar.com/lala1/lala1.html', - 'http://foobar.com/lala2.html', - 'http://foobar.com/pepepe/papapa/lala3.html', - 'http://foobar.com/imgs/lala4.jpg', - )) - self.assertEqual(sample_adaptor(sample_selector.x('//a[@onclick]').re(r'opensomething\(\'(.*?)\'\)')), ( - 'http://foobar.com/my_html1.html', - 'http://foobar.com/dummy/my_html2.html' - )) - - def test_to_unicode(self): - self.assertEqual(adaptors.to_unicode('lelelulu\xc3\xb1', {}), u'lelelulu\xf1') - self.assertEqual(adaptors.to_unicode(1, {}), u'1') - self.assertEqual(adaptors.to_unicode('\xc3\xa1\xc3\xa9', {'encoding': 'utf-8'}), u'\xe1\xe9') - self.assertEqual(adaptors.to_unicode('\xf1', {'encoding': 'latin1'}), u'\xf1') - - def test_regex(self): - adaptor = adaptors.Regex(regex=r'href="(.*?)"') - self.assertEqual(adaptor(['dsa', - 'href="lelelel.net"']), - ['lala.com', 'pepe.co.uk', 'das.biz', 'lelelel.net']) - - def test_unquote_all(self): - unquote = adaptors.unquote() - self.assertEqual(unquote(u'hello©&welcome<br />&', {}), - u'hello\xa9&welcome
&') - - def test_unquote(self): - unquote = adaptors.unquote(keep=['amp', 'lt']) - self.assertEqual(unquote(u'hello©&welcome<br />&', {}), - u'hello\xa9&welcome<br />&') - - def test_remove_tags(self): - self.assertEqual(adaptors.remove_tags()('adsaas
'), 'adsaas') - self.assertEqual(adaptors.remove_tags()('
dsadasf
'), 'dsadasf') - - def test_remove_root(self): - self.assertEqual(adaptors.remove_root('
lallaadsfsdfdspepepep
'), - 'lallaadsfsdfdspepepep
') - - def test_remove_multispaces(self): - self.assertEqual(adaptors.clean_spaces(' hello, whats up?'), ' hello, whats up?') - - def test_strip(self): - self.assertEqual(adaptors.strip(' hello there, this is my test '), - 'hello there, this is my test') - - def test_drop_empty_elements(self): - self.assertEqual(adaptors.drop_empty([1, 2, None, 5, 0, 6, False, 'hi']), - [1, 2, 5, 6, 'hi']) - - def test_delist(self): - delist = adaptors.delist(' ') - self.assertEqual(delist(['hi', 'there', 'fellas.', 'this', 'is', 'my', 'test.'], {}), - 'hi there fellas. this is my test.') - self.assertEqual(delist(['hi', 'there', 'fellas,', 'this', 'is', 'my', 'test.'], {'join_delimiter': '.'}), - 'hi.there.fellas,.this.is.my.test.') - - diff --git a/scrapy/tests/test_robustscrapeditem.py b/scrapy/tests/test_robustscrapeditem.py index 7cac5a45e..38e5eeb0d 100644 --- a/scrapy/tests/test_robustscrapeditem.py +++ b/scrapy/tests/test_robustscrapeditem.py @@ -2,8 +2,6 @@ import unittest from scrapy.contrib.item import RobustScrapedItem -from scrapy.contrib.item.adaptors import AdaptorPipe -from scrapy.contrib_exp import adaptors class MyItem(RobustScrapedItem): ATTRIBUTES = { @@ -55,37 +53,3 @@ class RobustScrapedItemTestCase(unittest.TestCase): self.item.attribute('children', 'Johnny', 'Rodrigo', add=True) self.assertEqual(self.item.children, ['Ken', 'Tom', 'Jimmy', 'Johnny', 'Rodrigo']) - def test_set_adaptors(self): - self.assertEqual(self.item._adaptors_dict, {}) - - delist = adaptors.delist() - self.item.set_adaptors({'name': [adaptors.extract, delist]}) - self.assertTrue(isinstance(self.item._adaptors_dict['name'], AdaptorPipe)) - self.assertEqual(self.item._adaptors_dict['name'][0].name, "extract") - self.assertEqual(self.item._adaptors_dict['name'][1].name, "delist") - - self.item.set_adaptors({'description': [adaptors.extract]}) - self.assertEqual(self.item._adaptors_dict['description'][0].name, "extract") - - def test_set_attrib_adaptors(self): - self.assertEqual(self.item._adaptors_dict, {}) - - self.item.set_attrib_adaptors('name', [adaptors.extract, adaptors.strip]) - self.assertTrue(isinstance(self.item._adaptors_dict['name'], AdaptorPipe)) - self.assertEqual(self.item._adaptors_dict['name'][0].name, "extract") - self.assertEqual(self.item._adaptors_dict['name'][1].name, "strip") - - unquote = adaptors.unquote() - self.item.set_attrib_adaptors('name', [adaptors.extract, unquote]) - self.assertTrue(isinstance(self.item._adaptors_dict['name'], AdaptorPipe)) - self.assertEqual(self.item._adaptors_dict['name'][0].name, "extract") - self.assertEqual(self.item._adaptors_dict['name'][1].name, "unquote") - - def test_add_adaptor(self): - self.assertEqual(self.item._adaptors_dict, {}) - - self.item.add_adaptor('name', adaptors.strip) - self.assertEqual(self.item._adaptors_dict['name'][0].name, "strip") - self.item.add_adaptor('name', adaptors.extract, position=0) - self.assertEqual(self.item._adaptors_dict['name'][0].name, "extract") - self.assertEqual(self.item._adaptors_dict['name'][1].name, "strip")