diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 790937af5..bb3164582 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -15,11 +15,17 @@ from .csstranslator import ScrapyHTMLTranslator, ScrapyGenericTranslator
__all__ = ['Selector', 'SelectorList']
+
+class SafeXMLParser(etree.XMLParser):
+ def __init__(self, *args, **kwargs):
+ kwargs.setdefault('resolve_entities', False)
+ super(SafeXMLParser, self).__init__(*args, **kwargs)
+
_ctgroup = {
'html': {'_parser': etree.HTMLParser,
'_csstranslator': ScrapyHTMLTranslator(),
'_tostring_method': 'html'},
- 'xml': {'_parser': etree.XMLParser,
+ 'xml': {'_parser': SafeXMLParser,
'_csstranslator': ScrapyGenericTranslator(),
'_tostring_method': 'xml'},
}
diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py
index d84e4bd47..710f1c28e 100644
--- a/scrapy/tests/test_selector.py
+++ b/scrapy/tests/test_selector.py
@@ -297,6 +297,16 @@ class SelectorTestCase(unittest.TestCase):
sel.remove_namespaces()
self.assertEqual(len(sel.xpath("//link/@type")), 2)
+ def test_xml_entity_expansion(self):
+ malicious_xml = ''\
+ ' ]>&xxe;'
+
+ response = XmlResponse('http://example.com', body=malicious_xml)
+ sel = self.sscls(response=response)
+
+ self.assertEqual(sel.extract(), '&xxe;')
+
class DeprecatedXpathSelectorTest(unittest.TestCase):
diff --git a/scrapy/tests/test_utils_sitemap.py b/scrapy/tests/test_utils_sitemap.py
index 0049e5c40..56585143f 100644
--- a/scrapy/tests/test_utils_sitemap.py
+++ b/scrapy/tests/test_utils_sitemap.py
@@ -188,13 +188,28 @@ Disallow: /forum/active/
""")
-
+
self.assertEqual(list(s), [
{'loc': 'http://www.example.com/english/',
'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
}
])
+ def test_xml_entity_expansion(self):
+ s = Sitemap("""
+
+
+ ]>
+
+
+ http://127.0.0.1:8000/&xxe;
+
+
+ """)
+
+ self.assertEqual(list(s), [{'loc': 'http://127.0.0.1:8000/'}])
+
if __name__ == '__main__':
unittest.main()
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 24a540531..bbf37bc28 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -12,7 +12,7 @@ class Sitemap(object):
(type=sitemapindex) files"""
def __init__(self, xmltext):
- xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True)
+ xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True, resolve_entities=False)
self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
rt = self._root.tag
self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt