scrapy.utils.sitemap: added one more case of parsing invalid sitemaps

scrapy/tests/test_utils_sitemap.py

@@ -84,6 +84,29 @@ class SitemapTest(unittest.TestCase):
              {'loc': 'http://www.example.com/2', 'lastmod': ''},
             ])
 
+    def test_sitemap_wrong_ns2(self):
+        """We have seen sitemaps with wrongs ns. Presumably, Google still works
+        with these, though is not 100% confirmed"""
+        s = Sitemap("""<?xml version="1.0" encoding="UTF-8"?>
+<urlset>
+  <url xmlns="">
+    <loc> http://www.example.com/</loc>
+    <lastmod>2009-08-16</lastmod>
+    <changefreq>daily</changefreq>
+    <priority>1</priority>
+  </url>
+  <url xmlns="">
+    <loc> http://www.example.com/2</loc>
+    <lastmod />
+  </url>
+</urlset>
+""")
+        assert s.type == 'urlset'
+        self.assertEqual(list(s),
+            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+             {'loc': 'http://www.example.com/2', 'lastmod': ''},
+            ])
+
     def test_sitemap_urls_from_robots(self):
         robots = """User-agent: *
 Disallow: /aff/

scrapy/utils/sitemap.py

@@ -16,7 +16,8 @@ class Sitemap(object):
         tree = ElementTree()
         tree.parse(StringIO(xmltext))
         self._root = tree.getroot()
-        _, self.type = self._root.tag.split('}', 1)
+        rt = self._root.tag
+        self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt
 
     def __iter__(self):
         for elem in self._root.getchildren():