Merge branch 'gh-347'. thanks @chrisboo

scrapy/tests/test_utils_sitemap.py

@@ -120,12 +120,45 @@ Disallow: /s*/*tags
 Sitemap: http://example.com/sitemap.xml
 Sitemap: http://example.com/sitemap-product-index.xml
 
-# Forums 
+# Forums
 Disallow: /forum/search/
 Disallow: /forum/active/
 """
-        self.assertEqual(list(sitemap_urls_from_robots(robots)), 
+        self.assertEqual(list(sitemap_urls_from_robots(robots)),
              ['http://example.com/sitemap.xml', 'http://example.com/sitemap-product-index.xml'])
 
+    def test_sitemap_blanklines(self):
+        """Assert we can deal with starting blank lines before <xml> tag"""
+        s = Sitemap("""\
+
+<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+
+<!-- cache: cached = yes name = sitemap_jspCache key = sitemap -->
+<sitemap>
+<loc>http://www.example.com/sitemap1.xml</loc>
+<lastmod>2013-07-15</lastmod>
+</sitemap>
+
+<sitemap>
+<loc>http://www.example.com/sitemap2.xml</loc>
+<lastmod>2013-07-15</lastmod>
+</sitemap>
+
+<sitemap>
+<loc>http://www.example.com/sitemap3.xml</loc>
+<lastmod>2013-07-15</lastmod>
+</sitemap>
+
+<!-- end cache -->
+</sitemapindex>
+""")
+        self.assertEqual(list(s), [
+            {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap1.xml'},
+            {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap2.xml'},
+            {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap3.xml'},
+        ])
+
+
 if __name__ == '__main__':
     unittest.main()

scrapy/utils/sitemap.py

@@ -4,18 +4,16 @@ Module for processing Sitemaps.
 Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
+import lxml.etree
 
-from cStringIO import StringIO
-from xml.etree.cElementTree import ElementTree
 
 class Sitemap(object):
     """Class to parse Sitemap (type=urlset) and Sitemap Index
     (type=sitemapindex) files"""
 
     def __init__(self, xmltext):
-        tree = ElementTree()
-        tree.parse(StringIO(xmltext))
-        self._root = tree.getroot()
+        xmlp = lxml.etree.XMLParser(recover=True)
+        self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
         rt = self._root.tag
         self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt
 
@@ -26,7 +24,9 @@ class Sitemap(object):
                 tag = el.tag
                 name = tag.split('}', 1)[1] if '}' in tag else tag
                 d[name] = el.text.strip() if el.text else ''
-            yield d
+            if 'loc' in d:
+                yield d
+
 
 def sitemap_urls_from_robots(robots_text):
     """Return an iterator over all sitemap urls contained in the given