RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove internal references to old selector classes and api

This commit is contained in:
Daniel Graña 2013-10-14 10:51:16 -02:00
parent 4e94b38396
commit add3506928
8 changed files with 37 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/contrib/linkextractors/sgml.py
View File

@ -4,7 +4,7 @@ SGMLParser-based Link extractors
import re
from urlparse import urlparse, urljoin
from w3lib.url import safe_url_string
from scrapy.selector import HtmlXPathSelector
from scrapy.selector import Selector
from scrapy.link import Link
from scrapy.linkextractor import IGNORED_EXTENSIONS
from scrapy.utils.misc import arg_to_iter
@ -116,11 +116,11 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor):
def extract_links(self, response):
base_url = None
if self.restrict_xpaths:
hxs = HtmlXPathSelector(response)
ss = Selector(response)
base_url = get_base_url(response)
body = u''.join(f
for x in self.restrict_xpaths
for f in hxs.select(x).extract()
for f in ss.xpath(x).extract()
).encode(response.encoding)
else:
body = response.body

7
scrapy/contrib/loader/__init__.py
View File

@ -8,7 +8,7 @@ from collections import defaultdict
import re
from scrapy.item import Item
from scrapy.selector import HtmlXPathSelector
from scrapy.selector import Selector
from scrapy.utils.misc import arg_to_iter, extract_regex
from scrapy.utils.python import flatten
from .common import wrap_loader_context
@ -116,7 +116,7 @@ class ItemLoader(object):
class XPathItemLoader(ItemLoader):
default_selector_class = HtmlXPathSelector
default_selector_class = Selector
def __init__(self, item=None, selector=None, response=None, **context):
if selector is None and response is None:
@ -142,5 +142,4 @@ class XPathItemLoader(ItemLoader):
def _get_values(self, xpaths, **kw):
xpaths = arg_to_iter(xpaths)
return flatten([self.selector.select(xpath).extract() for xpath in xpaths])
return flatten([self.selector.xpath(xpath).extract() for xpath in xpaths])

12
scrapy/contrib/spiders/feed.py
View File

@ -9,7 +9,7 @@ from scrapy.item import BaseItem
from scrapy.http import Request
from scrapy.utils.iterators import xmliter, csviter
from scrapy.utils.spider import iterate_spider_output
from scrapy.selector import XmlXPathSelector, HtmlXPathSelector
from scrapy.selector import Selector
from scrapy.exceptions import NotConfigured, NotSupported
@ -52,7 +52,7 @@ class XMLFeedSpider(BaseSpider):
def parse_nodes(self, response, nodes):
"""This method is called for the nodes matching the provided tag name
(itertag). Receives the response and an XPathSelector for each node.
(itertag). Receives the response and an Selector for each node.
Overriding this method is mandatory. Otherwise, you spider won't work.
This method must return either a BaseItem, a Request, or a list
containing any of them.
@ -71,13 +71,13 @@ class XMLFeedSpider(BaseSpider):
if self.iterator == 'iternodes':
nodes = self._iternodes(response)
elif self.iterator == 'xml':
selector = XmlXPathSelector(response)
selector = Selector(response, contenttype='xml')
self._register_namespaces(selector)
nodes = selector.select('//%s' % self.itertag)
nodes = selector.xpath('//%s' % self.itertag)
elif self.iterator == 'html':
selector = HtmlXPathSelector(response)
selector = Selector(response, contenttype='html')
self._register_namespaces(selector)
nodes = selector.select('//%s' % self.itertag)
nodes = selector.xpath('//%s' % self.itertag)
else:
raise NotSupported('Unsupported node iterator')

6
scrapy/contrib_exp/iterators.py
View File

@ -1,5 +1,5 @@
from scrapy.http import Response
from scrapy.selector import XmlXPathSelector
from scrapy.selector import Selector
def xmliter_lxml(obj, nodename, namespace=None):
@ -11,10 +11,10 @@ def xmliter_lxml(obj, nodename, namespace=None):
for _, node in iterable:
nodetext = etree.tostring(node)
node.clear()
xs = XmlXPathSelector(text=nodetext)
xs = Selector(text=nodetext, contenttype='xml')
if namespace:
xs.register_namespace('x', namespace)
yield xs.select(selxpath)[0]
yield xs.xpath(selxpath)[0]
class _StreamReader(object):

4
scrapy/tests/test_contrib_loader.py
View File

@ -4,7 +4,7 @@ from scrapy.contrib.loader import ItemLoader, XPathItemLoader
from scrapy.contrib.loader.processor import Join, Identity, TakeFirst, \
Compose, MapCompose
from scrapy.item import Item, Field
from scrapy.selector import HtmlXPathSelector
from scrapy.selector import Selector
from scrapy.http import HtmlResponse
@ -379,7 +379,7 @@ class XPathItemLoaderTest(unittest.TestCase):
self.assertRaises(RuntimeError, XPathItemLoader)
def test_constructor_with_selector(self):
sel = HtmlXPathSelector(text=u"<html><body><div>marta</div></body></html>")
sel = Selector(text=u"<html><body><div>marta</div></body></html>")
l = TestXPathItemLoader(selector=sel)
self.assert_(l.selector is sel)
l.add_xpath('name', '//div/text()')

8
scrapy/tests/test_spider.py
View File

@ -70,10 +70,10 @@ class XMLFeedSpiderTest(BaseSpiderTest):
def parse_node(self, response, selector):
yield {
'loc': selector.select('a:loc/text()').extract(),
'updated': selector.select('b:updated/text()').extract(),
'other': selector.select('other/@value').extract(),
'custom': selector.select('other/@b:custom').extract(),
'loc': selector.xpath('a:loc/text()').extract(),
'updated': selector.xpath('b:updated/text()').extract(),
'other': selector.xpath('other/@value').extract(),
'custom': selector.xpath('other/@b:custom').extract(),
}
for iterator in ('iternodes', 'xml'):

26
scrapy/tests/test_utils_iterators.py
View File

@ -28,7 +28,7 @@ class XmliterTestCase(unittest.TestCase):
response = XmlResponse(url="http://example.com", body=body)
attrs = []
for x in self.xmliter(response, 'product'):
attrs.append((x.select("@id").extract(), x.select("name/text()").extract(), x.select("./type/text()").extract()))
attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract()))
self.assertEqual(attrs,
[(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
@ -36,7 +36,7 @@ class XmliterTestCase(unittest.TestCase):
def test_xmliter_text(self):
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
self.assertEqual([x.select("text()").extract() for x in self.xmliter(body, 'product')],
self.assertEqual([x.xpath("text()").extract() for x in self.xmliter(body, 'product')],
[[u'one'], [u'two']])
def test_xmliter_namespaces(self):
@ -63,15 +63,15 @@ class XmliterTestCase(unittest.TestCase):
node = my_iter.next()
node.register_namespace('g', 'http://base.google.com/ns/1.0')
self.assertEqual(node.select('title/text()').extract(), ['Item 1'])
self.assertEqual(node.select('description/text()').extract(), ['This is item 1'])
self.assertEqual(node.select('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])
self.assertEqual(node.select('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
self.assertEqual(node.select('g:id/text()').extract(), ['ITEM_1'])
self.assertEqual(node.select('g:price/text()').extract(), ['400'])
self.assertEqual(node.select('image_link/text()').extract(), [])
self.assertEqual(node.select('id/text()').extract(), [])
self.assertEqual(node.select('price/text()').extract(), [])
self.assertEqual(node.xpath('title/text()').extract(), ['Item 1'])
self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1'])
self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])
self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1'])
self.assertEqual(node.xpath('g:price/text()').extract(), ['400'])
self.assertEqual(node.xpath('image_link/text()').extract(), [])
self.assertEqual(node.xpath('id/text()').extract(), [])
self.assertEqual(node.xpath('price/text()').extract(), [])
def test_xmliter_exception(self):
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
@ -123,9 +123,9 @@ class LxmlXmliterTestCase(XmliterTestCase):
namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0')
node = namespace_iter.next()
self.assertEqual(node.select('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
node = namespace_iter.next()
self.assertEqual(node.select('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg'])
self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg'])
class UtilsCsvTestCase(unittest.TestCase):

6
scrapy/utils/iterators.py
View File

@ -2,14 +2,14 @@ import re, csv
from cStringIO import StringIO
from scrapy.http import TextResponse
from scrapy.selector import XmlXPathSelector
from scrapy.selector import Selector
from scrapy import log
from scrapy.utils.python import re_rsearch, str_to_unicode
from scrapy.utils.response import body_or_str
def xmliter(obj, nodename):
"""Return a iterator of XPathSelector's over all nodes of a XML document,
"""Return a iterator of Selector's over all nodes of a XML document,
given tha name of the node to iterate. Useful for parsing XML feeds.
obj can be:
@ -29,7 +29,7 @@ def xmliter(obj, nodename):
r = re.compile(r"<%s[\s>].*?</%s>" % (nodename, nodename), re.DOTALL)
for match in r.finditer(text):
nodetext = header_start + match.group() + header_end
yield XmlXPathSelector(text=nodetext).select('//' + nodename)[0]
yield Selector(text=nodetext, contenttype='xml').xpath('//' + nodename)[0]
def csviter(obj, delimiter=None, headers=None, encoding=None):