mirror of https://github.com/scrapy/scrapy.git
Merge pull request #1409 from eliasdorneles/migrate-parsel
[MRG+1] Migrating selectors to use parsel
This commit is contained in:
Daniel Graña
commit
15c1300d35
|
|
@ -7,3 +7,4 @@ queuelib
|
|||
six>=1.5.2
|
||||
PyDispatcher>=2.0.5
|
||||
service_identity
|
||||
parsel>=0.9.5
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ See documentation in docs/topics/request-response.rst
|
|||
|
||||
from six.moves.urllib.parse import urljoin, urlencode
|
||||
import lxml.html
|
||||
from parsel.selector import create_root_node
|
||||
import six
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.utils.python import to_bytes, is_listlike
|
||||
|
|
@ -56,8 +57,8 @@ def _urlencode(seq, enc):
|
|||
|
||||
def _get_form(response, formname, formid, formnumber, formxpath):
|
||||
"""Find the form element """
|
||||
from scrapy.selector.lxmldocument import LxmlDocument
|
||||
root = LxmlDocument(response, lxml.html.HTMLParser)
|
||||
text = response.body_as_unicode()
|
||||
root = create_root_node(text, lxml.html.HTMLParser, base_url=response.url)
|
||||
forms = root.xpath('//form')
|
||||
if not forms:
|
||||
raise ValueError("No <form> element found in %s" % response)
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class LxmlParserLinkExtractor(object):
|
|||
def _extract_links(self, selector, response_url, response_encoding, base_url):
|
||||
links = []
|
||||
# hacky way to get the underlying lxml parsed document
|
||||
for el, attr, attr_val in self._iter_links(selector._root):
|
||||
for el, attr, attr_val in self._iter_links(selector.root):
|
||||
# pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
|
||||
attr_val = urljoin(base_url, attr_val)
|
||||
url = self.process_attr(attr_val)
|
||||
|
|
@ -65,9 +65,8 @@ class LxmlParserLinkExtractor(object):
|
|||
if self.unique else links
|
||||
|
||||
def extract_links(self, response):
|
||||
html = Selector(response)
|
||||
base_url = get_base_url(response)
|
||||
return self._extract_links(html, response.url, response.encoding, base_url)
|
||||
return self._extract_links(response.selector, response.url, response.encoding, base_url)
|
||||
|
||||
def _process_links(self, links):
|
||||
""" Normalize and filter extracted links
|
||||
|
|
@ -95,14 +94,13 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
|
|||
canonicalize=canonicalize, deny_extensions=deny_extensions)
|
||||
|
||||
def extract_links(self, response):
|
||||
html = Selector(response)
|
||||
base_url = get_base_url(response)
|
||||
if self.restrict_xpaths:
|
||||
docs = [subdoc
|
||||
for x in self.restrict_xpaths
|
||||
for subdoc in html.xpath(x)]
|
||||
for subdoc in response.xpath(x)]
|
||||
else:
|
||||
docs = [html]
|
||||
docs = [response.selector]
|
||||
all_links = []
|
||||
for doc in docs:
|
||||
links = self._extract_links(doc, response.url, response.encoding, base_url)
|
||||
|
|
|
|||
|
|
@ -127,11 +127,10 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
|
|||
def extract_links(self, response):
|
||||
base_url = None
|
||||
if self.restrict_xpaths:
|
||||
sel = Selector(response)
|
||||
base_url = get_base_url(response)
|
||||
body = u''.join(f
|
||||
for x in self.restrict_xpaths
|
||||
for f in sel.xpath(x).extract()
|
||||
for f in response.xpath(x).extract()
|
||||
).encode(response.encoding, errors='xmlcharrefreplace')
|
||||
else:
|
||||
body = response.body
|
||||
|
|
|
|||
|
|
@ -1,88 +1,15 @@
|
|||
from cssselect import GenericTranslator, HTMLTranslator
|
||||
from cssselect.xpath import _unicode_safe_getattr, XPathExpr, ExpressionError
|
||||
from cssselect.parser import FunctionalPseudoElement
|
||||
from parsel.csstranslator import XPathExpr, GenericTranslator, HTMLTranslator
|
||||
from scrapy.utils.deprecate import create_deprecated_class
|
||||
|
||||
|
||||
class ScrapyXPathExpr(XPathExpr):
|
||||
ScrapyXPathExpr = create_deprecated_class(
|
||||
'ScrapyXPathExpr', XPathExpr,
|
||||
new_class_path='parsel.csstranslator.XPathExpr')
|
||||
|
||||
textnode = False
|
||||
attribute = None
|
||||
|
||||
@classmethod
|
||||
def from_xpath(cls, xpath, textnode=False, attribute=None):
|
||||
x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition)
|
||||
x.textnode = textnode
|
||||
x.attribute = attribute
|
||||
return x
|
||||
|
||||
def __str__(self):
|
||||
path = super(ScrapyXPathExpr, self).__str__()
|
||||
if self.textnode:
|
||||
if path == '*':
|
||||
path = 'text()'
|
||||
elif path.endswith('::*/*'):
|
||||
path = path[:-3] + 'text()'
|
||||
else:
|
||||
path += '/text()'
|
||||
|
||||
if self.attribute is not None:
|
||||
if path.endswith('::*/*'):
|
||||
path = path[:-2]
|
||||
path += '/@%s' % self.attribute
|
||||
|
||||
return path
|
||||
|
||||
def join(self, combiner, other):
|
||||
super(ScrapyXPathExpr, self).join(combiner, other)
|
||||
self.textnode = other.textnode
|
||||
self.attribute = other.attribute
|
||||
return self
|
||||
|
||||
|
||||
class TranslatorMixin(object):
|
||||
|
||||
def xpath_element(self, selector):
|
||||
xpath = super(TranslatorMixin, self).xpath_element(selector)
|
||||
return ScrapyXPathExpr.from_xpath(xpath)
|
||||
|
||||
def xpath_pseudo_element(self, xpath, pseudo_element):
|
||||
if isinstance(pseudo_element, FunctionalPseudoElement):
|
||||
method = 'xpath_%s_functional_pseudo_element' % (
|
||||
pseudo_element.name.replace('-', '_'))
|
||||
method = _unicode_safe_getattr(self, method, None)
|
||||
if not method:
|
||||
raise ExpressionError(
|
||||
"The functional pseudo-element ::%s() is unknown"
|
||||
% pseudo_element.name)
|
||||
xpath = method(xpath, pseudo_element)
|
||||
else:
|
||||
method = 'xpath_%s_simple_pseudo_element' % (
|
||||
pseudo_element.replace('-', '_'))
|
||||
method = _unicode_safe_getattr(self, method, None)
|
||||
if not method:
|
||||
raise ExpressionError(
|
||||
"The pseudo-element ::%s is unknown"
|
||||
% pseudo_element)
|
||||
xpath = method(xpath)
|
||||
return xpath
|
||||
|
||||
def xpath_attr_functional_pseudo_element(self, xpath, function):
|
||||
if function.argument_types() not in (['STRING'], ['IDENT']):
|
||||
raise ExpressionError(
|
||||
"Expected a single string or ident for ::attr(), got %r"
|
||||
% function.arguments)
|
||||
return ScrapyXPathExpr.from_xpath(xpath,
|
||||
attribute=function.arguments[0].value)
|
||||
|
||||
def xpath_text_simple_pseudo_element(self, xpath):
|
||||
"""Support selecting text nodes using ::text pseudo-element"""
|
||||
return ScrapyXPathExpr.from_xpath(xpath, textnode=True)
|
||||
|
||||
|
||||
class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator):
|
||||
pass
|
||||
|
||||
|
||||
class ScrapyHTMLTranslator(TranslatorMixin, HTMLTranslator):
|
||||
pass
|
||||
ScrapyGenericTranslator = create_deprecated_class(
|
||||
'ScrapyGenericTranslator', GenericTranslator,
|
||||
new_class_path='parsel.csstranslator.GenericTranslator')
|
||||
|
||||
ScrapyHTMLTranslator = create_deprecated_class(
|
||||
'ScrapyHTMLTranslator', HTMLTranslator,
|
||||
new_class_path='parsel.csstranslator.HTMLTranslator')
|
||||
|
|
|
|||
|
|
@ -1,31 +0,0 @@
|
|||
"""
|
||||
This module contains a simple class (LxmlDocument) which provides cache and
|
||||
garbage collection to lxml element tree documents.
|
||||
"""
|
||||
|
||||
import weakref
|
||||
from lxml import etree
|
||||
from scrapy.utils.trackref import object_ref
|
||||
|
||||
|
||||
def _factory(response, parser_cls):
|
||||
url = response.url
|
||||
body = response.body_as_unicode().strip().encode('utf8') or '<html/>'
|
||||
parser = parser_cls(recover=True, encoding='utf8')
|
||||
return etree.fromstring(body, parser=parser, base_url=url)
|
||||
|
||||
|
||||
class LxmlDocument(object_ref):
|
||||
|
||||
cache = weakref.WeakKeyDictionary()
|
||||
__slots__ = ['__weakref__']
|
||||
|
||||
def __new__(cls, response, parser=etree.HTMLParser):
|
||||
cache = cls.cache.setdefault(response, {})
|
||||
if parser not in cache:
|
||||
obj = object_ref.__new__(cls)
|
||||
cache[parser] = _factory(response, parser)
|
||||
return cache[parser]
|
||||
|
||||
def __str__(self):
|
||||
return "<LxmlDocument %s>" % self.root.tag
|
||||
|
|
@ -2,43 +2,22 @@
|
|||
XPath selectors based on lxml
|
||||
"""
|
||||
|
||||
from lxml import etree
|
||||
import six
|
||||
|
||||
from scrapy.utils.misc import extract_regex
|
||||
import warnings
|
||||
from parsel import Selector as _ParselSelector
|
||||
from scrapy.utils.trackref import object_ref
|
||||
from scrapy.utils.python import to_bytes, flatten, iflatten
|
||||
from scrapy.utils.decorators import deprecated
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.http import HtmlResponse, XmlResponse
|
||||
from .lxmldocument import LxmlDocument
|
||||
from .csstranslator import ScrapyHTMLTranslator, ScrapyGenericTranslator
|
||||
from scrapy.utils.decorators import deprecated
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
__all__ = ['Selector', 'SelectorList']
|
||||
|
||||
|
||||
class SafeXMLParser(etree.XMLParser):
|
||||
def __init__(self, *args, **kwargs):
|
||||
kwargs.setdefault('resolve_entities', False)
|
||||
super(SafeXMLParser, self).__init__(*args, **kwargs)
|
||||
|
||||
_ctgroup = {
|
||||
'html': {'_parser': etree.HTMLParser,
|
||||
'_csstranslator': ScrapyHTMLTranslator(),
|
||||
'_tostring_method': 'html'},
|
||||
'xml': {'_parser': SafeXMLParser,
|
||||
'_csstranslator': ScrapyGenericTranslator(),
|
||||
'_tostring_method': 'xml'},
|
||||
}
|
||||
|
||||
|
||||
def _st(response, st):
|
||||
if st is None:
|
||||
return 'xml' if isinstance(response, XmlResponse) else 'html'
|
||||
elif st in ('xml', 'html'):
|
||||
return st
|
||||
else:
|
||||
raise ValueError('Invalid type: %s' % st)
|
||||
return st
|
||||
|
||||
|
||||
def _response_from_text(text, st):
|
||||
|
|
@ -47,149 +26,7 @@ def _response_from_text(text, st):
|
|||
body=to_bytes(text, 'utf-8'))
|
||||
|
||||
|
||||
class Selector(object_ref):
|
||||
|
||||
__slots__ = ['response', 'text', 'namespaces', 'type', '_expr', '_root',
|
||||
'__weakref__', '_parser', '_csstranslator', '_tostring_method']
|
||||
|
||||
_default_type = None
|
||||
_default_namespaces = {
|
||||
"re": "http://exslt.org/regular-expressions",
|
||||
|
||||
# supported in libxslt:
|
||||
# set:difference
|
||||
# set:has-same-node
|
||||
# set:intersection
|
||||
# set:leading
|
||||
# set:trailing
|
||||
"set": "http://exslt.org/sets"
|
||||
}
|
||||
_lxml_smart_strings = False
|
||||
|
||||
def __init__(self, response=None, text=None, type=None, namespaces=None,
|
||||
_root=None, _expr=None):
|
||||
self.type = st = _st(response, type or self._default_type)
|
||||
self._parser = _ctgroup[st]['_parser']
|
||||
self._csstranslator = _ctgroup[st]['_csstranslator']
|
||||
self._tostring_method = _ctgroup[st]['_tostring_method']
|
||||
|
||||
if text is not None:
|
||||
response = _response_from_text(text, st)
|
||||
|
||||
if response is not None:
|
||||
_root = LxmlDocument(response, self._parser)
|
||||
|
||||
self.response = response
|
||||
self.namespaces = dict(self._default_namespaces)
|
||||
if namespaces is not None:
|
||||
self.namespaces.update(namespaces)
|
||||
self._root = _root
|
||||
self._expr = _expr
|
||||
|
||||
def xpath(self, query):
|
||||
try:
|
||||
xpathev = self._root.xpath
|
||||
except AttributeError:
|
||||
return SelectorList([])
|
||||
|
||||
try:
|
||||
result = xpathev(query, namespaces=self.namespaces,
|
||||
smart_strings=self._lxml_smart_strings)
|
||||
except etree.XPathError:
|
||||
msg = u"Invalid XPath: %s" % query
|
||||
raise ValueError(msg if six.PY3 else msg.encode("unicode_escape"))
|
||||
|
||||
if type(result) is not list:
|
||||
result = [result]
|
||||
|
||||
result = [self.__class__(_root=x, _expr=query,
|
||||
namespaces=self.namespaces,
|
||||
type=self.type)
|
||||
for x in result]
|
||||
return SelectorList(result)
|
||||
|
||||
def css(self, query):
|
||||
return self.xpath(self._css2xpath(query))
|
||||
|
||||
def _css2xpath(self, query):
|
||||
return self._csstranslator.css_to_xpath(query)
|
||||
|
||||
def re(self, regex):
|
||||
return extract_regex(regex, self.extract())
|
||||
|
||||
def extract(self):
|
||||
try:
|
||||
return etree.tostring(self._root,
|
||||
method=self._tostring_method,
|
||||
encoding="unicode",
|
||||
with_tail=False)
|
||||
except (AttributeError, TypeError):
|
||||
if self._root is True:
|
||||
return u'1'
|
||||
elif self._root is False:
|
||||
return u'0'
|
||||
else:
|
||||
return six.text_type(self._root)
|
||||
|
||||
def register_namespace(self, prefix, uri):
|
||||
if self.namespaces is None:
|
||||
self.namespaces = {}
|
||||
self.namespaces[prefix] = uri
|
||||
|
||||
def remove_namespaces(self):
|
||||
for el in self._root.iter('*'):
|
||||
if el.tag.startswith('{'):
|
||||
el.tag = el.tag.split('}', 1)[1]
|
||||
# loop on element attributes also
|
||||
for an in el.attrib.keys():
|
||||
if an.startswith('{'):
|
||||
el.attrib[an.split('}', 1)[1]] = el.attrib.pop(an)
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.extract())
|
||||
|
||||
def __str__(self):
|
||||
data = repr(self.extract()[:40])
|
||||
return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data)
|
||||
__repr__ = __str__
|
||||
|
||||
# Deprecated api
|
||||
@deprecated(use_instead='.xpath()')
|
||||
def select(self, xpath):
|
||||
return self.xpath(xpath)
|
||||
|
||||
@deprecated(use_instead='.extract()')
|
||||
def extract_unquoted(self):
|
||||
return self.extract()
|
||||
|
||||
|
||||
class SelectorList(list):
|
||||
|
||||
def __getslice__(self, i, j):
|
||||
return self.__class__(list.__getslice__(self, i, j))
|
||||
|
||||
def xpath(self, xpath):
|
||||
return self.__class__(flatten([x.xpath(xpath) for x in self]))
|
||||
|
||||
def css(self, xpath):
|
||||
return self.__class__(flatten([x.css(xpath) for x in self]))
|
||||
|
||||
def re(self, regex):
|
||||
return flatten([x.re(regex) for x in self])
|
||||
|
||||
def re_first(self, regex):
|
||||
for el in iflatten(x.re(regex) for x in self):
|
||||
return el
|
||||
|
||||
def extract(self):
|
||||
return [x.extract() for x in self]
|
||||
|
||||
def extract_first(self, default=None):
|
||||
for x in self:
|
||||
return x.extract()
|
||||
else:
|
||||
return default
|
||||
|
||||
class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
|
||||
@deprecated(use_instead='.extract()')
|
||||
def extract_unquoted(self):
|
||||
return [x.extract_unquoted() for x in self]
|
||||
|
|
@ -201,3 +38,45 @@ class SelectorList(list):
|
|||
@deprecated(use_instead='.xpath()')
|
||||
def select(self, xpath):
|
||||
return self.xpath(xpath)
|
||||
|
||||
|
||||
class Selector(_ParselSelector, object_ref):
|
||||
|
||||
__slots__ = ['response']
|
||||
selectorlist_cls = SelectorList
|
||||
|
||||
def __init__(self, response=None, text=None, type=None, root=None, _root=None, **kwargs):
|
||||
st = _st(response, type or self._default_type)
|
||||
|
||||
if _root is not None:
|
||||
warnings.warn("Argument `_root` is deprecated, use `root` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
if root is None:
|
||||
root = _root
|
||||
else:
|
||||
warnings.warn("Ignoring deprecated `_root` argument, using provided `root`")
|
||||
|
||||
if text is not None:
|
||||
response = _response_from_text(text, st)
|
||||
|
||||
if response is not None:
|
||||
text = response.body_as_unicode()
|
||||
kwargs.setdefault('base_url', response.url)
|
||||
|
||||
self.response = response
|
||||
super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
|
||||
|
||||
# Deprecated api
|
||||
@property
|
||||
def _root(self):
|
||||
warnings.warn("Attribute `_root` is deprecated, use `root` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return self.root
|
||||
|
||||
@deprecated(use_instead='.xpath()')
|
||||
def select(self, xpath):
|
||||
return self.xpath(xpath)
|
||||
|
||||
@deprecated(use_instead='.extract()')
|
||||
def extract_unquoted(self):
|
||||
return self.extract()
|
||||
|
|
|
|||
1
setup.py
1
setup.py
|
|
@ -44,6 +44,7 @@ setup(
|
|||
'pyOpenSSL',
|
||||
'cssselect>=0.9',
|
||||
'six>=1.5.2',
|
||||
'parsel>=0.9.3',
|
||||
'PyDispatcher>=2.0.5',
|
||||
'service_identity',
|
||||
],
|
||||
|
|
|
|||
|
|
@ -25,9 +25,6 @@ tests/test_mail.py
|
|||
tests/test_pipeline_files.py
|
||||
tests/test_pipeline_images.py
|
||||
tests/test_proxy_connect.py
|
||||
tests/test_selector_csstranslator.py
|
||||
tests/test_selector_lxmldocument.py
|
||||
tests/test_selector.py
|
||||
tests/test_spidermiddleware_depth.py
|
||||
tests/test_spidermiddleware_httperror.py
|
||||
tests/test_spidermiddleware_offsite.py
|
||||
|
|
|
|||
|
|
@ -1,28 +1,24 @@
|
|||
import re
|
||||
import warnings
|
||||
import weakref
|
||||
import six
|
||||
from twisted.trial import unittest
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import TextResponse, HtmlResponse, XmlResponse
|
||||
from scrapy.selector import Selector
|
||||
from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector
|
||||
from lxml import etree
|
||||
|
||||
|
||||
class SelectorTestCase(unittest.TestCase):
|
||||
|
||||
sscls = Selector
|
||||
|
||||
def test_simple_selection(self):
|
||||
"""Simple selector tests"""
|
||||
body = "<p><input name='a'value='1'/><input name='b'value='2'/></p>"
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
sel = self.sscls(response)
|
||||
body = b"<p><input name='a'value='1'/><input name='b'value='2'/></p>"
|
||||
response = TextResponse(url="http://example.com", body=body, encoding='utf-8')
|
||||
sel = Selector(response)
|
||||
|
||||
xl = sel.xpath('//input')
|
||||
self.assertEqual(2, len(xl))
|
||||
for x in xl:
|
||||
assert isinstance(x, self.sscls)
|
||||
assert isinstance(x, Selector)
|
||||
|
||||
self.assertEqual(sel.xpath('//input').extract(),
|
||||
[x.extract() for x in sel.xpath('//input')])
|
||||
|
|
@ -37,234 +33,41 @@ class SelectorTestCase(unittest.TestCase):
|
|||
self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
|
||||
[u'12'])
|
||||
|
||||
def test_representation_slice(self):
|
||||
body = u"<p><input name='{}' value='\xa9'/></p>".format(50 * 'b')
|
||||
response = TextResponse(url="http://example.com", body=body, encoding='utf8')
|
||||
sel = self.sscls(response)
|
||||
def test_root_base_url(self):
|
||||
body = b'<html><form action="/path"><input name="a" /></form></html>'
|
||||
url = "http://example.com"
|
||||
response = TextResponse(url=url, body=body, encoding='utf-8')
|
||||
sel = Selector(response)
|
||||
self.assertEqual(url, sel.root.base)
|
||||
|
||||
self.assertEqual(
|
||||
map(repr, sel.xpath('//input/@name')),
|
||||
["<Selector xpath='//input/@name' data=u'{}'>".format(40 * 'b')]
|
||||
)
|
||||
def test_deprecated_root_argument(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
root = etree.fromstring(u'<html/>')
|
||||
sel = Selector(_root=root)
|
||||
self.assertIs(root, sel.root)
|
||||
self.assertEqual(str(w[-1].message),
|
||||
'Argument `_root` is deprecated, use `root` instead')
|
||||
|
||||
def test_representation_unicode_query(self):
|
||||
body = u"<p><input name='{}' value='\xa9'/></p>".format(50 * 'b')
|
||||
response = TextResponse(url="http://example.com", body=body, encoding='utf8')
|
||||
sel = self.sscls(response)
|
||||
self.assertEqual(
|
||||
map(repr, sel.xpath(u'//input[@value="\xa9"]/@value')),
|
||||
["<Selector xpath=u'//input[@value=\"\\xa9\"]/@value' data=u'\\xa9'>"]
|
||||
)
|
||||
|
||||
def test_extract_first(self):
|
||||
"""Test if extract_first() returns first element"""
|
||||
body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
sel = self.sscls(response)
|
||||
|
||||
self.assertEqual(sel.xpath('//ul/li/text()').extract_first(),
|
||||
sel.xpath('//ul/li/text()').extract()[0])
|
||||
|
||||
self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').extract_first(),
|
||||
sel.xpath('//ul/li[@id="1"]/text()').extract()[0])
|
||||
|
||||
self.assertEqual(sel.xpath('//ul/li[2]/text()').extract_first(),
|
||||
sel.xpath('//ul/li/text()').extract()[1])
|
||||
|
||||
self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').extract_first(), None)
|
||||
|
||||
def test_extract_first_default(self):
|
||||
"""Test if extract_first() returns default value when no results found"""
|
||||
body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
sel = self.sscls(response)
|
||||
|
||||
self.assertEqual(sel.xpath('//div/text()').extract_first(default='missing'), 'missing')
|
||||
|
||||
def test_re_first(self):
|
||||
"""Test if re_first() returns first matched element"""
|
||||
body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
sel = self.sscls(response)
|
||||
|
||||
self.assertEqual(sel.xpath('//ul/li/text()').re_first('\d'),
|
||||
sel.xpath('//ul/li/text()').re('\d')[0])
|
||||
|
||||
self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').re_first('\d'),
|
||||
sel.xpath('//ul/li[@id="1"]/text()').re('\d')[0])
|
||||
|
||||
self.assertEqual(sel.xpath('//ul/li[2]/text()').re_first('\d'),
|
||||
sel.xpath('//ul/li/text()').re('\d')[1])
|
||||
|
||||
self.assertEqual(sel.xpath('/ul/li/text()').re_first('\w+'), None)
|
||||
self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').re_first('\d'), None)
|
||||
|
||||
def test_select_unicode_query(self):
|
||||
body = u"<p><input name='\xa9' value='1'/></p>"
|
||||
response = TextResponse(url="http://example.com", body=body, encoding='utf8')
|
||||
sel = self.sscls(response)
|
||||
self.assertEqual(sel.xpath(u'//input[@name="\xa9"]/@value').extract(), [u'1'])
|
||||
|
||||
def test_list_elements_type(self):
|
||||
"""Test Selector returning the same type in selection methods"""
|
||||
text = '<p>test<p>'
|
||||
assert isinstance(self.sscls(text=text).xpath("//p")[0], self.sscls)
|
||||
assert isinstance(self.sscls(text=text).css("p")[0], self.sscls)
|
||||
|
||||
def test_boolean_result(self):
|
||||
body = "<p><input name='a'value='1'/><input name='b'value='2'/></p>"
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
xs = self.sscls(response)
|
||||
self.assertEquals(xs.xpath("//input[@name='a']/@name='a'").extract(), [u'1'])
|
||||
self.assertEquals(xs.xpath("//input[@name='a']/@name='n'").extract(), [u'0'])
|
||||
|
||||
def test_differences_parsing_xml_vs_html(self):
|
||||
"""Test that XML and HTML Selector's behave differently"""
|
||||
# some text which is parsed differently by XML and HTML flavors
|
||||
text = '<div><img src="a.jpg"><p>Hello</div>'
|
||||
hs = self.sscls(text=text, type='html')
|
||||
self.assertEqual(hs.xpath("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
|
||||
|
||||
xs = self.sscls(text=text, type='xml')
|
||||
self.assertEqual(xs.xpath("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
|
||||
def test_deprecated_root_argument_ambiguous(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
_root = etree.fromstring(u'<xml/>')
|
||||
root = etree.fromstring(u'<html/>')
|
||||
sel = Selector(_root=_root, root=root)
|
||||
self.assertIs(root, sel.root)
|
||||
self.assertIn('Ignoring deprecated `_root` argument', str(w[-1].message))
|
||||
|
||||
def test_flavor_detection(self):
|
||||
text = '<div><img src="a.jpg"><p>Hello</div>'
|
||||
sel = self.sscls(XmlResponse('http://example.com', body=text))
|
||||
text = b'<div><img src="a.jpg"><p>Hello</div>'
|
||||
sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
|
||||
self.assertEqual(sel.type, 'xml')
|
||||
self.assertEqual(sel.xpath("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
|
||||
|
||||
sel = self.sscls(HtmlResponse('http://example.com', body=text))
|
||||
sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
|
||||
self.assertEqual(sel.type, 'html')
|
||||
self.assertEqual(sel.xpath("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
|
||||
|
||||
def test_nested_selectors(self):
|
||||
"""Nested selector tests"""
|
||||
body = """<body>
|
||||
<div class='one'>
|
||||
<ul>
|
||||
<li>one</li><li>two</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class='two'>
|
||||
<ul>
|
||||
<li>four</li><li>five</li><li>six</li>
|
||||
</ul>
|
||||
</div>
|
||||
</body>"""
|
||||
|
||||
response = HtmlResponse(url="http://example.com", body=body)
|
||||
x = self.sscls(response)
|
||||
divtwo = x.xpath('//div[@class="two"]')
|
||||
self.assertEqual(divtwo.xpath("//li").extract(),
|
||||
["<li>one</li>", "<li>two</li>", "<li>four</li>", "<li>five</li>", "<li>six</li>"])
|
||||
self.assertEqual(divtwo.xpath("./ul/li").extract(),
|
||||
["<li>four</li>", "<li>five</li>", "<li>six</li>"])
|
||||
self.assertEqual(divtwo.xpath(".//li").extract(),
|
||||
["<li>four</li>", "<li>five</li>", "<li>six</li>"])
|
||||
self.assertEqual(divtwo.xpath("./li").extract(), [])
|
||||
|
||||
def test_mixed_nested_selectors(self):
|
||||
body = '''<body>
|
||||
<div id=1>not<span>me</span></div>
|
||||
<div class="dos"><p>text</p><a href='#'>foo</a></div>
|
||||
</body>'''
|
||||
sel = self.sscls(text=body)
|
||||
self.assertEqual(sel.xpath('//div[@id="1"]').css('span::text').extract(), [u'me'])
|
||||
self.assertEqual(sel.css('#1').xpath('./span/text()').extract(), [u'me'])
|
||||
|
||||
def test_dont_strip(self):
|
||||
sel = self.sscls(text='<div>fff: <a href="#">zzz</a></div>')
|
||||
self.assertEqual(sel.xpath("//text()").extract(), [u'fff: ', u'zzz'])
|
||||
|
||||
def test_namespaces_simple(self):
|
||||
body = """
|
||||
<test xmlns:somens="http://scrapy.org">
|
||||
<somens:a id="foo">take this</a>
|
||||
<a id="bar">found</a>
|
||||
</test>
|
||||
"""
|
||||
|
||||
response = XmlResponse(url="http://example.com", body=body)
|
||||
x = self.sscls(response)
|
||||
|
||||
x.register_namespace("somens", "http://scrapy.org")
|
||||
self.assertEqual(x.xpath("//somens:a/text()").extract(),
|
||||
[u'take this'])
|
||||
|
||||
def test_namespaces_multiple(self):
|
||||
body = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<BrowseNode xmlns="http://webservices.amazon.com/AWSECommerceService/2005-10-05"
|
||||
xmlns:b="http://somens.com"
|
||||
xmlns:p="http://www.scrapy.org/product" >
|
||||
<b:Operation>hello</b:Operation>
|
||||
<TestTag b:att="value"><Other>value</Other></TestTag>
|
||||
<p:SecondTestTag><material>iron</material><price>90</price><p:name>Dried Rose</p:name></p:SecondTestTag>
|
||||
</BrowseNode>
|
||||
"""
|
||||
response = XmlResponse(url="http://example.com", body=body)
|
||||
x = self.sscls(response)
|
||||
x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05")
|
||||
x.register_namespace("p", "http://www.scrapy.org/product")
|
||||
x.register_namespace("b", "http://somens.com")
|
||||
self.assertEqual(len(x.xpath("//xmlns:TestTag")), 1)
|
||||
self.assertEqual(x.xpath("//b:Operation/text()").extract()[0], 'hello')
|
||||
self.assertEqual(x.xpath("//xmlns:TestTag/@b:att").extract()[0], 'value')
|
||||
self.assertEqual(x.xpath("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90')
|
||||
self.assertEqual(x.xpath("//p:SecondTestTag").xpath("./xmlns:price/text()")[0].extract(), '90')
|
||||
self.assertEqual(x.xpath("//p:SecondTestTag/xmlns:material/text()").extract()[0], 'iron')
|
||||
|
||||
def test_re(self):
|
||||
body = """<div>Name: Mary
|
||||
<ul>
|
||||
<li>Name: John</li>
|
||||
<li>Age: 10</li>
|
||||
<li>Name: Paul</li>
|
||||
<li>Age: 20</li>
|
||||
</ul>
|
||||
Age: 20
|
||||
</div>"""
|
||||
response = HtmlResponse(url="http://example.com", body=body)
|
||||
x = self.sscls(response)
|
||||
|
||||
name_re = re.compile("Name: (\w+)")
|
||||
self.assertEqual(x.xpath("//ul/li").re(name_re),
|
||||
["John", "Paul"])
|
||||
self.assertEqual(x.xpath("//ul/li").re("Age: (\d+)"),
|
||||
["10", "20"])
|
||||
|
||||
def test_re_intl(self):
|
||||
body = """<div>Evento: cumplea\xc3\xb1os</div>"""
|
||||
response = HtmlResponse(url="http://example.com", body=body, encoding='utf-8')
|
||||
x = self.sscls(response)
|
||||
self.assertEqual(x.xpath("//div").re("Evento: (\w+)"), [u'cumplea\xf1os'])
|
||||
|
||||
def test_selector_over_text(self):
|
||||
hs = self.sscls(text='<root>lala</root>')
|
||||
self.assertEqual(hs.extract(), u'<html><body><root>lala</root></body></html>')
|
||||
xs = self.sscls(text='<root>lala</root>', type='xml')
|
||||
self.assertEqual(xs.extract(), u'<root>lala</root>')
|
||||
self.assertEqual(xs.xpath('.').extract(), [u'<root>lala</root>'])
|
||||
|
||||
def test_invalid_xpath(self):
|
||||
"Test invalid xpath raises ValueError with the invalid xpath"
|
||||
response = XmlResponse(url="http://example.com", body="<html></html>")
|
||||
x = self.sscls(response)
|
||||
xpath = "//test[@foo='bar]"
|
||||
self.assertRaisesRegexp(ValueError, re.escape(xpath), x.xpath, xpath)
|
||||
|
||||
def test_invalid_xpath_unicode(self):
|
||||
"Test *Unicode* invalid xpath raises ValueError with the invalid xpath"
|
||||
response = XmlResponse(url="http://example.com", body="<html></html>")
|
||||
x = self.sscls(response)
|
||||
xpath = u"//test[@foo='\u0431ar]"
|
||||
encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
|
||||
self.assertRaisesRegexp(ValueError, re.escape(encoded), x.xpath, xpath)
|
||||
|
||||
def test_http_header_encoding_precedence(self):
|
||||
# u'\xa3' = pound symbol in unicode
|
||||
# u'\xc2\xa3' = pound symbol in utf-8
|
||||
|
|
@ -280,131 +83,45 @@ class SelectorTestCase(unittest.TestCase):
|
|||
|
||||
headers = {'Content-Type': ['text/html; charset=utf-8']}
|
||||
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
|
||||
x = self.sscls(response)
|
||||
x = Selector(response)
|
||||
self.assertEquals(x.xpath("//span[@id='blank']/text()").extract(),
|
||||
[u'\xa3'])
|
||||
|
||||
def test_empty_bodies(self):
|
||||
# shouldn't raise errors
|
||||
r1 = TextResponse('http://www.example.com', body='')
|
||||
self.sscls(r1).xpath('//text()').extract()
|
||||
|
||||
def test_null_bytes(self):
|
||||
# shouldn't raise errors
|
||||
r1 = TextResponse('http://www.example.com', \
|
||||
body='<root>pre\x00post</root>', \
|
||||
encoding='utf-8')
|
||||
self.sscls(r1).xpath('//text()').extract()
|
||||
|
||||
def test_badly_encoded_body(self):
|
||||
# \xe9 alone isn't valid utf8 sequence
|
||||
r1 = TextResponse('http://www.example.com', \
|
||||
body='<html><p>an Jos\xe9 de</p><html>', \
|
||||
body=b'<html><p>an Jos\xe9 de</p><html>', \
|
||||
encoding='utf-8')
|
||||
self.sscls(r1).xpath('//text()').extract()
|
||||
|
||||
def test_select_on_unevaluable_nodes(self):
|
||||
r = self.sscls(text=u'<span class="big">some text</span>')
|
||||
# Text node
|
||||
x1 = r.xpath('//text()')
|
||||
self.assertEquals(x1.extract(), [u'some text'])
|
||||
self.assertEquals(x1.xpath('.//b').extract(), [])
|
||||
# Tag attribute
|
||||
x1 = r.xpath('//span/@class')
|
||||
self.assertEquals(x1.extract(), [u'big'])
|
||||
self.assertEquals(x1.xpath('.//text()').extract(), [])
|
||||
|
||||
def test_select_on_text_nodes(self):
|
||||
r = self.sscls(text=u'<div><b>Options:</b>opt1</div><div><b>Other</b>opt2</div>')
|
||||
x1 = r.xpath("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]")
|
||||
self.assertEquals(x1.extract(), [u'opt1'])
|
||||
|
||||
x1 = r.xpath("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]")
|
||||
self.assertEquals(x1.extract(), [u'<b>Options:</b>'])
|
||||
|
||||
def test_nested_select_on_text_nodes(self):
|
||||
# FIXME: does not work with lxml backend [upstream]
|
||||
r = self.sscls(text=u'<div><b>Options:</b>opt1</div><div><b>Other</b>opt2</div>')
|
||||
x1 = r.xpath("//div/descendant::text()")
|
||||
x2 = x1.xpath("./preceding-sibling::b[contains(text(), 'Options')]")
|
||||
self.assertEquals(x2.extract(), [u'<b>Options:</b>'])
|
||||
test_nested_select_on_text_nodes.skip = "Text nodes lost parent node reference in lxml"
|
||||
Selector(r1).xpath('//text()').extract()
|
||||
|
||||
def test_weakref_slots(self):
|
||||
"""Check that classes are using slots and are weak-referenceable"""
|
||||
x = self.sscls()
|
||||
x = Selector(text='')
|
||||
weakref.ref(x)
|
||||
assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
|
||||
x.__class__.__name__
|
||||
|
||||
def test_remove_namespaces(self):
|
||||
xml = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en-US" xmlns:media="http://search.yahoo.com/mrss/">
|
||||
<link type="text/html">
|
||||
<link type="application/atom+xml">
|
||||
</feed>
|
||||
"""
|
||||
sel = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml))
|
||||
self.assertEqual(len(sel.xpath("//link")), 0)
|
||||
sel.remove_namespaces()
|
||||
self.assertEqual(len(sel.xpath("//link")), 2)
|
||||
def test_deprecated_selector_methods(self):
|
||||
sel = Selector(TextResponse(url="http://example.com", body=b'<p>some text</p>'))
|
||||
|
||||
def test_remove_attributes_namespaces(self):
|
||||
xml = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns:atom="http://www.w3.org/2005/Atom" xml:lang="en-US" xmlns:media="http://search.yahoo.com/mrss/">
|
||||
<link atom:type="text/html">
|
||||
<link atom:type="application/atom+xml">
|
||||
</feed>
|
||||
"""
|
||||
sel = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml))
|
||||
self.assertEqual(len(sel.xpath("//link/@type")), 0)
|
||||
sel.remove_namespaces()
|
||||
self.assertEqual(len(sel.xpath("//link/@type")), 2)
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
sel.select('//p')
|
||||
self.assertSubstring('Use .xpath() instead', str(w[-1].message))
|
||||
|
||||
def test_smart_strings(self):
|
||||
"""Lxml smart strings return values"""
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
sel.extract_unquoted()
|
||||
self.assertSubstring('Use .extract() instead', str(w[-1].message))
|
||||
|
||||
class SmartStringsSelector(Selector):
|
||||
_lxml_smart_strings = True
|
||||
def test_deprecated_selectorlist_methods(self):
|
||||
sel = Selector(TextResponse(url="http://example.com", body=b'<p>some text</p>'))
|
||||
|
||||
body = """<body>
|
||||
<div class='one'>
|
||||
<ul>
|
||||
<li>one</li><li>two</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class='two'>
|
||||
<ul>
|
||||
<li>four</li><li>five</li><li>six</li>
|
||||
</ul>
|
||||
</div>
|
||||
</body>"""
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
sel.xpath('//p').select('.')
|
||||
self.assertSubstring('Use .xpath() instead', str(w[-1].message))
|
||||
|
||||
response = HtmlResponse(url="http://example.com", body=body)
|
||||
|
||||
# .getparent() is available for text nodes and attributes
|
||||
# only when smart_strings are on
|
||||
x = self.sscls(response)
|
||||
li_text = x.xpath('//li/text()')
|
||||
self.assertFalse(any(map(lambda e: hasattr(e._root, 'getparent'), li_text)))
|
||||
div_class = x.xpath('//div/@class')
|
||||
self.assertFalse(any(map(lambda e: hasattr(e._root, 'getparent'), div_class)))
|
||||
|
||||
x = SmartStringsSelector(response)
|
||||
li_text = x.xpath('//li/text()')
|
||||
self.assertTrue(all(map(lambda e: hasattr(e._root, 'getparent'), li_text)))
|
||||
div_class = x.xpath('//div/@class')
|
||||
self.assertTrue(all(map(lambda e: hasattr(e._root, 'getparent'), div_class)))
|
||||
|
||||
def test_xml_entity_expansion(self):
|
||||
malicious_xml = '<?xml version="1.0" encoding="ISO-8859-1"?>'\
|
||||
'<!DOCTYPE foo [ <!ELEMENT foo ANY > <!ENTITY xxe SYSTEM '\
|
||||
'"file:///etc/passwd" >]><foo>&xxe;</foo>'
|
||||
|
||||
response = XmlResponse('http://example.com', body=malicious_xml)
|
||||
sel = self.sscls(response=response)
|
||||
|
||||
self.assertEqual(sel.extract(), '<foo>&xxe;</foo>')
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
sel.xpath('//p').extract_unquoted()
|
||||
self.assertSubstring('Use .extract() instead', str(w[-1].message))
|
||||
|
||||
|
||||
class DeprecatedXpathSelectorTest(unittest.TestCase):
|
||||
|
|
@ -488,146 +205,3 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
|
|||
self.assertTrue(isinstance(usel, Selector))
|
||||
self.assertTrue(isinstance(sel, XPathSelector))
|
||||
self.assertTrue(isinstance(usel, XPathSelector))
|
||||
|
||||
def test_xpathselector(self):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
|
||||
hs = XPathSelector(text=self.text)
|
||||
self.assertEqual(hs.select("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
|
||||
self.assertRaises(RuntimeError, hs.css, 'div')
|
||||
|
||||
def test_htmlxpathselector(self):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
|
||||
hs = HtmlXPathSelector(text=self.text)
|
||||
self.assertEqual(hs.select("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
|
||||
self.assertRaises(RuntimeError, hs.css, 'div')
|
||||
|
||||
def test_xmlxpathselector(self):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
|
||||
xs = XmlXPathSelector(text=self.text)
|
||||
self.assertEqual(xs.select("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
|
||||
self.assertRaises(RuntimeError, xs.css, 'div')
|
||||
|
||||
|
||||
class ExsltTestCase(unittest.TestCase):
|
||||
|
||||
sscls = Selector
|
||||
|
||||
def test_regexp(self):
|
||||
"""EXSLT regular expression tests"""
|
||||
body = """
|
||||
<p><input name='a' value='1'/><input name='b' value='2'/></p>
|
||||
<div class="links">
|
||||
<a href="/first.html">first link</a>
|
||||
<a href="/second.html">second link</a>
|
||||
<a href="http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml">EXSLT match example</a>
|
||||
</div>
|
||||
"""
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
sel = self.sscls(response)
|
||||
|
||||
# re:test()
|
||||
self.assertEqual(
|
||||
sel.xpath(
|
||||
'//input[re:test(@name, "[A-Z]+", "i")]').extract(),
|
||||
[x.extract() for x in sel.xpath('//input[re:test(@name, "[A-Z]+", "i")]')])
|
||||
self.assertEqual(
|
||||
[x.extract()
|
||||
for x in sel.xpath(
|
||||
'//a[re:test(@href, "\.html$")]/text()')],
|
||||
[u'first link', u'second link'])
|
||||
self.assertEqual(
|
||||
[x.extract()
|
||||
for x in sel.xpath(
|
||||
'//a[re:test(@href, "first")]/text()')],
|
||||
[u'first link'])
|
||||
self.assertEqual(
|
||||
[x.extract()
|
||||
for x in sel.xpath(
|
||||
'//a[re:test(@href, "second")]/text()')],
|
||||
[u'second link'])
|
||||
|
||||
|
||||
# re:match() is rather special: it returns a node-set of <match> nodes
|
||||
#[u'<match>http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml</match>',
|
||||
#u'<match>http</match>',
|
||||
#u'<match>www.bayes.co.uk</match>',
|
||||
#u'<match></match>',
|
||||
#u'<match>/xml/index.xml?/xml/utils/rechecker.xml</match>']
|
||||
self.assertEqual(
|
||||
sel.xpath('re:match(//a[re:test(@href, "\.xml$")]/@href,'
|
||||
'"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(),
|
||||
[u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml',
|
||||
u'http',
|
||||
u'www.bayes.co.uk',
|
||||
u'',
|
||||
u'/xml/index.xml?/xml/utils/rechecker.xml'])
|
||||
|
||||
|
||||
|
||||
# re:replace()
|
||||
self.assertEqual(
|
||||
sel.xpath('re:replace(//a[re:test(@href, "\.xml$")]/@href,'
|
||||
'"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(),
|
||||
[u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html'])
|
||||
|
||||
def test_set(self):
|
||||
"""EXSLT set manipulation tests"""
|
||||
# microdata example from http://schema.org/Event
|
||||
body="""
|
||||
<div itemscope itemtype="http://schema.org/Event">
|
||||
<a itemprop="url" href="nba-miami-philidelphia-game3.html">
|
||||
NBA Eastern Conference First Round Playoff Tickets:
|
||||
<span itemprop="name"> Miami Heat at Philadelphia 76ers - Game 3 (Home Game 1) </span>
|
||||
</a>
|
||||
|
||||
<meta itemprop="startDate" content="2016-04-21T20:00">
|
||||
Thu, 04/21/16
|
||||
8:00 p.m.
|
||||
|
||||
<div itemprop="location" itemscope itemtype="http://schema.org/Place">
|
||||
<a itemprop="url" href="wells-fargo-center.html">
|
||||
Wells Fargo Center
|
||||
</a>
|
||||
<div itemprop="address" itemscope itemtype="http://schema.org/PostalAddress">
|
||||
<span itemprop="addressLocality">Philadelphia</span>,
|
||||
<span itemprop="addressRegion">PA</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div itemprop="offers" itemscope itemtype="http://schema.org/AggregateOffer">
|
||||
Priced from: <span itemprop="lowPrice">$35</span>
|
||||
<span itemprop="offerCount">1938</span> tickets left
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
sel = self.sscls(response)
|
||||
|
||||
self.assertEqual(
|
||||
sel.xpath('''//div[@itemtype="http://schema.org/Event"]
|
||||
//@itemprop''').extract(),
|
||||
[u'url',
|
||||
u'name',
|
||||
u'startDate',
|
||||
u'location',
|
||||
u'url',
|
||||
u'address',
|
||||
u'addressLocality',
|
||||
u'addressRegion',
|
||||
u'offers',
|
||||
u'lowPrice',
|
||||
u'offerCount']
|
||||
)
|
||||
|
||||
self.assertEqual(sel.xpath('''
|
||||
set:difference(//div[@itemtype="http://schema.org/Event"]
|
||||
//@itemprop,
|
||||
//div[@itemtype="http://schema.org/Event"]
|
||||
//*[@itemscope]/*/@itemprop)''').extract(),
|
||||
[u'url', u'name', u'startDate', u'location', u'offers'])
|
||||
|
|
|
|||
|
|
@ -1,153 +1,22 @@
|
|||
"""
|
||||
Selector tests for cssselect backend
|
||||
"""
|
||||
import warnings
|
||||
from twisted.trial import unittest
|
||||
from scrapy.http import HtmlResponse
|
||||
from scrapy.selector.csstranslator import ScrapyHTMLTranslator
|
||||
from scrapy.selector import Selector
|
||||
from cssselect.parser import SelectorSyntaxError
|
||||
from cssselect.xpath import ExpressionError
|
||||
from scrapy.selector.csstranslator import (
|
||||
ScrapyHTMLTranslator,
|
||||
ScrapyGenericTranslator,
|
||||
ScrapyXPathExpr
|
||||
)
|
||||
|
||||
|
||||
HTMLBODY = b'''
|
||||
<html>
|
||||
<body>
|
||||
<div>
|
||||
<a id="name-anchor" name="foo"></a>
|
||||
<a id="tag-anchor" rel="tag" href="http://localhost/foo">link</a>
|
||||
<a id="nofollow-anchor" rel="nofollow" href="https://example.org"> link</a>
|
||||
<p id="paragraph">
|
||||
lorem ipsum text
|
||||
<b id="p-b">hi</b> <em id="p-em">there</em>
|
||||
<b id="p-b2">guy</b>
|
||||
<input type="checkbox" id="checkbox-unchecked" />
|
||||
<input type="checkbox" id="checkbox-disabled" disabled="" />
|
||||
<input type="text" id="text-checked" checked="checked" />
|
||||
<input type="hidden" />
|
||||
<input type="hidden" disabled="disabled" />
|
||||
<input type="checkbox" id="checkbox-checked" checked="checked" />
|
||||
<input type="checkbox" id="checkbox-disabled-checked"
|
||||
disabled="disabled" checked="checked" />
|
||||
<fieldset id="fieldset" disabled="disabled">
|
||||
<input type="checkbox" id="checkbox-fieldset-disabled" />
|
||||
<input type="hidden" />
|
||||
</fieldset>
|
||||
</p>
|
||||
<map name="dummymap">
|
||||
<area shape="circle" coords="200,250,25" href="foo.html" id="area-href" />
|
||||
<area shape="default" id="area-nohref" />
|
||||
</map>
|
||||
</div>
|
||||
<div class="cool-footer" id="foobar-div" foobar="ab bc cde">
|
||||
<span id="foobar-span">foo ter</span>
|
||||
</div>
|
||||
</body></html>
|
||||
'''
|
||||
class DeprecatedClassesTest(unittest.TestCase):
|
||||
|
||||
def test_deprecated_warnings(self):
|
||||
for cls in [ScrapyHTMLTranslator, ScrapyGenericTranslator, ScrapyXPathExpr]:
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
obj = cls()
|
||||
self.assertIn('%s is deprecated' % cls.__name__, str(w[-1].message),
|
||||
'Missing deprecate warning for %s' % cls.__name__)
|
||||
|
||||
|
||||
class TranslatorMixinTest(unittest.TestCase):
|
||||
|
||||
tr_cls = ScrapyHTMLTranslator
|
||||
|
||||
def setUp(self):
|
||||
self.tr = self.tr_cls()
|
||||
self.c2x = self.tr.css_to_xpath
|
||||
|
||||
def test_attr_function(self):
|
||||
cases = [
|
||||
('::attr(name)', u'descendant-or-self::*/@name'),
|
||||
('a::attr(href)', u'descendant-or-self::a/@href'),
|
||||
('a ::attr(img)', u'descendant-or-self::a/descendant-or-self::*/@img'),
|
||||
('a > ::attr(class)', u'descendant-or-self::a/*/@class'),
|
||||
]
|
||||
for css, xpath in cases:
|
||||
self.assertEqual(self.c2x(css), xpath, css)
|
||||
|
||||
def test_attr_function_exception(self):
|
||||
cases = [
|
||||
('::attr(12)', ExpressionError),
|
||||
('::attr(34test)', ExpressionError),
|
||||
('::attr(@href)', SelectorSyntaxError),
|
||||
]
|
||||
for css, exc in cases:
|
||||
self.assertRaises(exc, self.c2x, css)
|
||||
|
||||
def test_text_pseudo_element(self):
|
||||
cases = [
|
||||
('::text', u'descendant-or-self::text()'),
|
||||
('p::text', u'descendant-or-self::p/text()'),
|
||||
('p ::text', u'descendant-or-self::p/descendant-or-self::text()'),
|
||||
('#id::text', u"descendant-or-self::*[@id = 'id']/text()"),
|
||||
('p#id::text', u"descendant-or-self::p[@id = 'id']/text()"),
|
||||
('p#id ::text', u"descendant-or-self::p[@id = 'id']/descendant-or-self::text()"),
|
||||
('p#id > ::text', u"descendant-or-self::p[@id = 'id']/*/text()"),
|
||||
('p#id ~ ::text', u"descendant-or-self::p[@id = 'id']/following-sibling::*/text()"),
|
||||
('a[href]::text', u'descendant-or-self::a[@href]/text()'),
|
||||
('a[href] ::text', u'descendant-or-self::a[@href]/descendant-or-self::text()'),
|
||||
('p::text, a::text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"),
|
||||
]
|
||||
for css, xpath in cases:
|
||||
self.assertEqual(self.c2x(css), xpath, css)
|
||||
|
||||
def test_pseudo_function_exception(self):
|
||||
cases = [
|
||||
('::attribute(12)', ExpressionError),
|
||||
('::text()', ExpressionError),
|
||||
('::attr(@href)', SelectorSyntaxError),
|
||||
]
|
||||
for css, exc in cases:
|
||||
self.assertRaises(exc, self.c2x, css)
|
||||
|
||||
def test_unknown_pseudo_element(self):
|
||||
cases = [
|
||||
('::text-node', ExpressionError),
|
||||
]
|
||||
for css, exc in cases:
|
||||
self.assertRaises(exc, self.c2x, css)
|
||||
|
||||
def test_unknown_pseudo_class(self):
|
||||
cases = [
|
||||
(':text', ExpressionError),
|
||||
(':attribute(name)', ExpressionError),
|
||||
]
|
||||
for css, exc in cases:
|
||||
self.assertRaises(exc, self.c2x, css)
|
||||
|
||||
|
||||
class CSSSelectorTest(unittest.TestCase):
|
||||
|
||||
sscls = Selector
|
||||
|
||||
def setUp(self):
|
||||
self.htmlresponse = HtmlResponse('http://example.com', body=HTMLBODY)
|
||||
self.sel = self.sscls(self.htmlresponse)
|
||||
|
||||
def x(self, *a, **kw):
|
||||
return [v.strip() for v in self.sel.css(*a, **kw).extract() if v.strip()]
|
||||
|
||||
def test_selector_simple(self):
|
||||
for x in self.sel.css('input'):
|
||||
self.assertTrue(isinstance(x, self.sel.__class__), x)
|
||||
self.assertEqual(self.sel.css('input').extract(),
|
||||
[x.extract() for x in self.sel.css('input')])
|
||||
|
||||
def test_text_pseudo_element(self):
|
||||
self.assertEqual(self.x('#p-b2'), [u'<b id="p-b2">guy</b>'])
|
||||
self.assertEqual(self.x('#p-b2::text'), [u'guy'])
|
||||
self.assertEqual(self.x('#p-b2 ::text'), [u'guy'])
|
||||
self.assertEqual(self.x('#paragraph::text'), [u'lorem ipsum text'])
|
||||
self.assertEqual(self.x('#paragraph ::text'), [u'lorem ipsum text', u'hi', u'there', u'guy'])
|
||||
self.assertEqual(self.x('p::text'), [u'lorem ipsum text'])
|
||||
self.assertEqual(self.x('p ::text'), [u'lorem ipsum text', u'hi', u'there', u'guy'])
|
||||
|
||||
def test_attribute_function(self):
|
||||
self.assertEqual(self.x('#p-b2::attr(id)'), [u'p-b2'])
|
||||
self.assertEqual(self.x('.cool-footer::attr(class)'), [u'cool-footer'])
|
||||
self.assertEqual(self.x('.cool-footer ::attr(id)'), [u'foobar-div', u'foobar-span'])
|
||||
self.assertEqual(self.x('map[name="dummymap"] ::attr(shape)'), [u'circle', u'default'])
|
||||
|
||||
def test_nested_selector(self):
|
||||
self.assertEqual(self.sel.css('p').css('b::text').extract(),
|
||||
[u'hi', u'guy'])
|
||||
self.assertEqual(self.sel.css('div').css('area:last-child').extract(),
|
||||
[u'<area shape="default" id="area-nohref">'])
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
import unittest
|
||||
from scrapy.selector.lxmldocument import LxmlDocument
|
||||
from scrapy.http import TextResponse, HtmlResponse
|
||||
|
||||
|
||||
class LxmlDocumentTest(unittest.TestCase):
|
||||
|
||||
def test_caching(self):
|
||||
r1 = HtmlResponse('http://www.example.com', body=b'<html><head></head><body></body></html>')
|
||||
r2 = r1.copy()
|
||||
|
||||
doc1 = LxmlDocument(r1)
|
||||
doc2 = LxmlDocument(r1)
|
||||
doc3 = LxmlDocument(r2)
|
||||
|
||||
# make sure it's cached
|
||||
assert doc1 is doc2
|
||||
assert doc1 is not doc3
|
||||
|
||||
def test_null_char(self):
|
||||
# make sure bodies with null char ('\x00') don't raise a TypeError exception
|
||||
body = b'test problematic \x00 body'
|
||||
response = TextResponse('http://example.com/catalog/product/blabla-123',
|
||||
headers={'Content-Type': 'text/plain; charset=utf-8'},
|
||||
body=body)
|
||||
LxmlDocument(response)
|
||||
Loading…
Reference in New Issue