diff --git a/scrapy/tests/test_utils_sitemap.py b/scrapy/tests/test_utils_sitemap.py index 99b66feec..8be94a309 100644 --- a/scrapy/tests/test_utils_sitemap.py +++ b/scrapy/tests/test_utils_sitemap.py @@ -120,12 +120,45 @@ Disallow: /s*/*tags Sitemap: http://example.com/sitemap.xml Sitemap: http://example.com/sitemap-product-index.xml -# Forums +# Forums Disallow: /forum/search/ Disallow: /forum/active/ """ - self.assertEqual(list(sitemap_urls_from_robots(robots)), + self.assertEqual(list(sitemap_urls_from_robots(robots)), ['http://example.com/sitemap.xml', 'http://example.com/sitemap-product-index.xml']) + def test_sitemap_blanklines(self): + """Assert we can deal with starting blank lines before tag""" + s = Sitemap("""\ + + + + + + +http://www.example.com/sitemap1.xml +2013-07-15 + + + +http://www.example.com/sitemap2.xml +2013-07-15 + + + +http://www.example.com/sitemap3.xml +2013-07-15 + + + + +""") + self.assertEqual(list(s), [ + {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap1.xml'}, + {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap2.xml'}, + {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap3.xml'}, + ]) + + if __name__ == '__main__': unittest.main() diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 71d8122ab..38e38d6a9 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,18 +4,16 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ +import lxml.etree -from cStringIO import StringIO -from xml.etree.cElementTree import ElementTree class Sitemap(object): """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" def __init__(self, xmltext): - tree = ElementTree() - tree.parse(StringIO(xmltext)) - self._root = tree.getroot() + xmlp = lxml.etree.XMLParser(recover=True) + self._root = lxml.etree.fromstring(xmltext, parser=xmlp) rt = self._root.tag self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt @@ -26,7 +24,9 @@ class Sitemap(object): tag = el.tag name = tag.split('}', 1)[1] if '}' in tag else tag d[name] = el.text.strip() if el.text else '' - yield d + if 'loc' in d: + yield d + def sitemap_urls_from_robots(robots_text): """Return an iterator over all sitemap urls contained in the given