diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 7f2f4e4bc..f0e141a43 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -4,7 +4,7 @@ SGMLParser-based Link extractors import re from urlparse import urlparse, urljoin from w3lib.url import safe_url_string -from scrapy.selector import HtmlXPathSelector +from scrapy.selector import Selector from scrapy.link import Link from scrapy.linkextractor import IGNORED_EXTENSIONS from scrapy.utils.misc import arg_to_iter @@ -116,11 +116,11 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor): def extract_links(self, response): base_url = None if self.restrict_xpaths: - hxs = HtmlXPathSelector(response) + ss = Selector(response) base_url = get_base_url(response) body = u''.join(f for x in self.restrict_xpaths - for f in hxs.select(x).extract() + for f in ss.xpath(x).extract() ).encode(response.encoding) else: body = response.body diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py index fa8387eaa..ecd20fc3d 100644 --- a/scrapy/contrib/loader/__init__.py +++ b/scrapy/contrib/loader/__init__.py @@ -8,7 +8,7 @@ from collections import defaultdict import re from scrapy.item import Item -from scrapy.selector import HtmlXPathSelector +from scrapy.selector import Selector from scrapy.utils.misc import arg_to_iter, extract_regex from scrapy.utils.python import flatten from .common import wrap_loader_context @@ -116,7 +116,7 @@ class ItemLoader(object): class XPathItemLoader(ItemLoader): - default_selector_class = HtmlXPathSelector + default_selector_class = Selector def __init__(self, item=None, selector=None, response=None, **context): if selector is None and response is None: @@ -142,5 +142,4 @@ class XPathItemLoader(ItemLoader): def _get_values(self, xpaths, **kw): xpaths = arg_to_iter(xpaths) - return flatten([self.selector.select(xpath).extract() for xpath in xpaths]) - + return flatten([self.selector.xpath(xpath).extract() for xpath in xpaths]) diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py index 89c6277ab..6d8e0a358 100644 --- a/scrapy/contrib/spiders/feed.py +++ b/scrapy/contrib/spiders/feed.py @@ -9,7 +9,7 @@ from scrapy.item import BaseItem from scrapy.http import Request from scrapy.utils.iterators import xmliter, csviter from scrapy.utils.spider import iterate_spider_output -from scrapy.selector import XmlXPathSelector, HtmlXPathSelector +from scrapy.selector import Selector from scrapy.exceptions import NotConfigured, NotSupported @@ -52,7 +52,7 @@ class XMLFeedSpider(BaseSpider): def parse_nodes(self, response, nodes): """This method is called for the nodes matching the provided tag name - (itertag). Receives the response and an XPathSelector for each node. + (itertag). Receives the response and an Selector for each node. Overriding this method is mandatory. Otherwise, you spider won't work. This method must return either a BaseItem, a Request, or a list containing any of them. @@ -71,13 +71,13 @@ class XMLFeedSpider(BaseSpider): if self.iterator == 'iternodes': nodes = self._iternodes(response) elif self.iterator == 'xml': - selector = XmlXPathSelector(response) + selector = Selector(response, contenttype='xml') self._register_namespaces(selector) - nodes = selector.select('//%s' % self.itertag) + nodes = selector.xpath('//%s' % self.itertag) elif self.iterator == 'html': - selector = HtmlXPathSelector(response) + selector = Selector(response, contenttype='html') self._register_namespaces(selector) - nodes = selector.select('//%s' % self.itertag) + nodes = selector.xpath('//%s' % self.itertag) else: raise NotSupported('Unsupported node iterator') diff --git a/scrapy/contrib_exp/iterators.py b/scrapy/contrib_exp/iterators.py index 0f3a8c694..ecc6265d0 100644 --- a/scrapy/contrib_exp/iterators.py +++ b/scrapy/contrib_exp/iterators.py @@ -1,5 +1,5 @@ from scrapy.http import Response -from scrapy.selector import XmlXPathSelector +from scrapy.selector import Selector def xmliter_lxml(obj, nodename, namespace=None): @@ -11,10 +11,10 @@ def xmliter_lxml(obj, nodename, namespace=None): for _, node in iterable: nodetext = etree.tostring(node) node.clear() - xs = XmlXPathSelector(text=nodetext) + xs = Selector(text=nodetext, contenttype='xml') if namespace: xs.register_namespace('x', namespace) - yield xs.select(selxpath)[0] + yield xs.xpath(selxpath)[0] class _StreamReader(object): diff --git a/scrapy/tests/test_contrib_loader.py b/scrapy/tests/test_contrib_loader.py index 80f3151da..976df5f12 100644 --- a/scrapy/tests/test_contrib_loader.py +++ b/scrapy/tests/test_contrib_loader.py @@ -4,7 +4,7 @@ from scrapy.contrib.loader import ItemLoader, XPathItemLoader from scrapy.contrib.loader.processor import Join, Identity, TakeFirst, \ Compose, MapCompose from scrapy.item import Item, Field -from scrapy.selector import HtmlXPathSelector +from scrapy.selector import Selector from scrapy.http import HtmlResponse @@ -379,7 +379,7 @@ class XPathItemLoaderTest(unittest.TestCase): self.assertRaises(RuntimeError, XPathItemLoader) def test_constructor_with_selector(self): - sel = HtmlXPathSelector(text=u"
marta
") + sel = Selector(text=u"
marta
") l = TestXPathItemLoader(selector=sel) self.assert_(l.selector is sel) l.add_xpath('name', '//div/text()') diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py index 9ffa2d7cc..97076089b 100644 --- a/scrapy/tests/test_spider.py +++ b/scrapy/tests/test_spider.py @@ -70,10 +70,10 @@ class XMLFeedSpiderTest(BaseSpiderTest): def parse_node(self, response, selector): yield { - 'loc': selector.select('a:loc/text()').extract(), - 'updated': selector.select('b:updated/text()').extract(), - 'other': selector.select('other/@value').extract(), - 'custom': selector.select('other/@b:custom').extract(), + 'loc': selector.xpath('a:loc/text()').extract(), + 'updated': selector.xpath('b:updated/text()').extract(), + 'other': selector.xpath('other/@value').extract(), + 'custom': selector.xpath('other/@b:custom').extract(), } for iterator in ('iternodes', 'xml'): diff --git a/scrapy/tests/test_utils_iterators.py b/scrapy/tests/test_utils_iterators.py index 21963031b..4ddd05693 100644 --- a/scrapy/tests/test_utils_iterators.py +++ b/scrapy/tests/test_utils_iterators.py @@ -28,7 +28,7 @@ class XmliterTestCase(unittest.TestCase): response = XmlResponse(url="http://example.com", body=body) attrs = [] for x in self.xmliter(response, 'product'): - attrs.append((x.select("@id").extract(), x.select("name/text()").extract(), x.select("./type/text()").extract())) + attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract())) self.assertEqual(attrs, [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) @@ -36,7 +36,7 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter_text(self): body = u"""onetwo""" - self.assertEqual([x.select("text()").extract() for x in self.xmliter(body, 'product')], + self.assertEqual([x.xpath("text()").extract() for x in self.xmliter(body, 'product')], [[u'one'], [u'two']]) def test_xmliter_namespaces(self): @@ -63,15 +63,15 @@ class XmliterTestCase(unittest.TestCase): node = my_iter.next() node.register_namespace('g', 'http://base.google.com/ns/1.0') - self.assertEqual(node.select('title/text()').extract(), ['Item 1']) - self.assertEqual(node.select('description/text()').extract(), ['This is item 1']) - self.assertEqual(node.select('link/text()').extract(), ['http://www.mydummycompany.com/items/1']) - self.assertEqual(node.select('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) - self.assertEqual(node.select('g:id/text()').extract(), ['ITEM_1']) - self.assertEqual(node.select('g:price/text()').extract(), ['400']) - self.assertEqual(node.select('image_link/text()').extract(), []) - self.assertEqual(node.select('id/text()').extract(), []) - self.assertEqual(node.select('price/text()').extract(), []) + self.assertEqual(node.xpath('title/text()').extract(), ['Item 1']) + self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1']) + self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1']) + self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1']) + self.assertEqual(node.xpath('g:price/text()').extract(), ['400']) + self.assertEqual(node.xpath('image_link/text()').extract(), []) + self.assertEqual(node.xpath('id/text()').extract(), []) + self.assertEqual(node.xpath('price/text()').extract(), []) def test_xmliter_exception(self): body = u"""onetwo""" @@ -123,9 +123,9 @@ class LxmlXmliterTestCase(XmliterTestCase): namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0') node = namespace_iter.next() - self.assertEqual(node.select('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) node = namespace_iter.next() - self.assertEqual(node.select('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg']) + self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg']) class UtilsCsvTestCase(unittest.TestCase): diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 327b9e024..67f9946e6 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -2,14 +2,14 @@ import re, csv from cStringIO import StringIO from scrapy.http import TextResponse -from scrapy.selector import XmlXPathSelector +from scrapy.selector import Selector from scrapy import log from scrapy.utils.python import re_rsearch, str_to_unicode from scrapy.utils.response import body_or_str def xmliter(obj, nodename): - """Return a iterator of XPathSelector's over all nodes of a XML document, + """Return a iterator of Selector's over all nodes of a XML document, given tha name of the node to iterate. Useful for parsing XML feeds. obj can be: @@ -29,7 +29,7 @@ def xmliter(obj, nodename): r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end - yield XmlXPathSelector(text=nodetext).select('//' + nodename)[0] + yield Selector(text=nodetext, contenttype='xml').xpath('//' + nodename)[0] def csviter(obj, delimiter=None, headers=None, encoding=None):