diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 790937af5..bb3164582 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -15,11 +15,17 @@ from .csstranslator import ScrapyHTMLTranslator, ScrapyGenericTranslator __all__ = ['Selector', 'SelectorList'] + +class SafeXMLParser(etree.XMLParser): + def __init__(self, *args, **kwargs): + kwargs.setdefault('resolve_entities', False) + super(SafeXMLParser, self).__init__(*args, **kwargs) + _ctgroup = { 'html': {'_parser': etree.HTMLParser, '_csstranslator': ScrapyHTMLTranslator(), '_tostring_method': 'html'}, - 'xml': {'_parser': etree.XMLParser, + 'xml': {'_parser': SafeXMLParser, '_csstranslator': ScrapyGenericTranslator(), '_tostring_method': 'xml'}, } diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index d84e4bd47..710f1c28e 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -297,6 +297,16 @@ class SelectorTestCase(unittest.TestCase): sel.remove_namespaces() self.assertEqual(len(sel.xpath("//link/@type")), 2) + def test_xml_entity_expansion(self): + malicious_xml = ''\ + ' ]>&xxe;' + + response = XmlResponse('http://example.com', body=malicious_xml) + sel = self.sscls(response=response) + + self.assertEqual(sel.extract(), '&xxe;') + class DeprecatedXpathSelectorTest(unittest.TestCase): diff --git a/scrapy/tests/test_utils_sitemap.py b/scrapy/tests/test_utils_sitemap.py index 0049e5c40..56585143f 100644 --- a/scrapy/tests/test_utils_sitemap.py +++ b/scrapy/tests/test_utils_sitemap.py @@ -188,13 +188,28 @@ Disallow: /forum/active/ """) - + self.assertEqual(list(s), [ {'loc': 'http://www.example.com/english/', 'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/'] } ]) + def test_xml_entity_expansion(self): + s = Sitemap(""" + + + ]> + + + http://127.0.0.1:8000/&xxe; + + + """) + + self.assertEqual(list(s), [{'loc': 'http://127.0.0.1:8000/'}]) + if __name__ == '__main__': unittest.main() diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 24a540531..bbf37bc28 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -12,7 +12,7 @@ class Sitemap(object): (type=sitemapindex) files""" def __init__(self, xmltext): - xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True) + xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True, resolve_entities=False) self._root = lxml.etree.fromstring(xmltext, parser=xmlp) rt = self._root.tag self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt