From d66efb13badcaf7939b1779d02b28db3f5ab65a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Wed, 7 Oct 2015 14:43:47 +0300 Subject: [PATCH 1/2] test xml nodename with dots --- tests/test_utils_iterators.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index f2780dcf1..590c53302 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -33,6 +33,19 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(attrs, [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) + def test_xmliter_unusual_node(self): + body = b""" + + + + + """ + response = XmlResponse(url="http://example.com", body=body) + nodenames = [e.xpath('name()').extract() + for e in self.xmliter(response, 'matchme...')] + self.assertEqual(nodenames, [['matchme...']]) + + def test_xmliter_text(self): body = u"""onetwo""" From f56062d04549e9906b9f824a7dbfa063df2b8abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Wed, 7 Oct 2015 14:47:23 +0300 Subject: [PATCH 2/2] escape nodename in xmliter regex --- scrapy/utils/iterators.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index b9f682431..c0d93f7a9 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -25,8 +25,10 @@ def xmliter(obj, nodename): - a unicode string - a string encoded as utf-8 """ - HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S) - HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S) + nodename_patt = re.escape(nodename) + + HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename_patt, re.S) + HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename_patt, re.S) text = _body_or_str(obj) header_start = re.search(HEADER_START_RE, text) @@ -34,7 +36,7 @@ def xmliter(obj, nodename): header_end = re_rsearch(HEADER_END_RE, text) header_end = text[header_end[1]:].strip() if header_end else '' - r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) + r = re.compile(r"<{0}[\s>].*?".format(nodename_patt), re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]