Merge pull request #694 from kmike/0.22

backport xml fixes to 0.22

scrapy/selector/unified.py

@@ -15,11 +15,17 @@ from .csstranslator import ScrapyHTMLTranslator, ScrapyGenericTranslator
 
 __all__ = ['Selector', 'SelectorList']
 
+
+class SafeXMLParser(etree.XMLParser):
+    def __init__(self, *args, **kwargs):
+        kwargs.setdefault('resolve_entities', False)
+        super(SafeXMLParser, self).__init__(*args, **kwargs)
+
 _ctgroup = {
     'html': {'_parser': etree.HTMLParser,
              '_csstranslator': ScrapyHTMLTranslator(),
              '_tostring_method': 'html'},
-    'xml': {'_parser': etree.XMLParser,
+    'xml': {'_parser': SafeXMLParser,
             '_csstranslator': ScrapyGenericTranslator(),
             '_tostring_method': 'xml'},
 }

scrapy/tests/test_selector.py

@@ -297,6 +297,16 @@ class SelectorTestCase(unittest.TestCase):
         sel.remove_namespaces()
         self.assertEqual(len(sel.xpath("//link/@type")), 2)
 
+    def test_xml_entity_expansion(self):
+        malicious_xml = '<?xml version="1.0" encoding="ISO-8859-1"?>'\
+            '<!DOCTYPE foo [ <!ELEMENT foo ANY > <!ENTITY xxe SYSTEM '\
+            '"file:///etc/passwd" >]><foo>&xxe;</foo>'
+
+        response = XmlResponse('http://example.com', body=malicious_xml)
+        sel = self.sscls(response=response)
+
+        self.assertEqual(sel.extract(), '<foo>&xxe;</foo>')
+
 
 class DeprecatedXpathSelectorTest(unittest.TestCase):
 

scrapy/tests/test_utils_sitemap.py

@@ -188,13 +188,28 @@ Disallow: /forum/active/
             <xhtml:link rel="alternate" hreflang="en"/><!-- wrong tag without href -->
         </url>
     </urlset>""")
-        
+
         self.assertEqual(list(s), [
             {'loc': 'http://www.example.com/english/',
              'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
             }
         ])
 
+    def test_xml_entity_expansion(self):
+        s = Sitemap("""<?xml version="1.0" encoding="utf-8"?>
+          <!DOCTYPE foo [
+          <!ELEMENT foo ANY >
+          <!ENTITY xxe SYSTEM "file:///etc/passwd" >
+          ]>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url>
+              <loc>http://127.0.0.1:8000/&xxe;</loc>
+            </url>
+          </urlset>
+        """)
+
+        self.assertEqual(list(s), [{'loc': 'http://127.0.0.1:8000/'}])
+
 
 if __name__ == '__main__':
     unittest.main()

scrapy/utils/sitemap.py

@@ -12,7 +12,7 @@ class Sitemap(object):
     (type=sitemapindex) files"""
 
     def __init__(self, xmltext):
-        xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True)
+        xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True, resolve_entities=False)
         self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
         rt = self._root.tag
         self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt