mirror of https://github.com/scrapy/scrapy.git
fixed minor bug in sitemap parser
This commit is contained in:
parent
ab9b786791
commit
574b070bb4
|
|
@ -51,10 +51,16 @@ class SitemapTest(unittest.TestCase):
|
|||
<changefreq>daily</changefreq>
|
||||
<priority>1</priority>
|
||||
</url>
|
||||
<url>
|
||||
<loc> http://www.example.com/2</loc>
|
||||
<lastmod />
|
||||
</url>
|
||||
</urlset>
|
||||
""")
|
||||
self.assertEqual(list(s),
|
||||
[{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}])
|
||||
[{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
|
||||
{'loc': 'http://www.example.com/2', 'lastmod': ''},
|
||||
])
|
||||
|
||||
def test_sitemap_urls_from_robots(self):
|
||||
robots = """User-agent: *
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ class Sitemap(object):
|
|||
d = {}
|
||||
for el in elem.getchildren():
|
||||
_, name = el.tag.split('}', 1)
|
||||
d[name] = el.text.strip()
|
||||
d[name] = el.text.strip() if el.text else ''
|
||||
yield d
|
||||
|
||||
def sitemap_urls_from_robots(robots_text):
|
||||
|
|
|
|||
Loading…
Reference in New Issue