diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index a08dc30f2..742a88659 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -680,6 +680,50 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. + .. method:: sitemap_filter(entries) + + This is a filter funtion that could be overridden to select sitemap entries + based on their attributes. + + For example:: + + + http://example.com/ + 2005-01-01 + + + We can define a ``sitemap_filter`` function to filter ``entries`` by date:: + + from datetime import datetime + from scrapy.spiders import SitemapSpider + + class FilteredSitemapSpider(SitemapSpider): + name = 'filtered_sitemap_spider' + allowed_domains = ['example.com'] + sitemap_urls = ['http://example.com/sitemap.xml'] + + def sitemap_filter(self, entries): + for entry in entries: + date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') + if date_time.year >= 2005: + yield entry + + This would retrieve only ``entries`` modified on 2005 and the following + years. + + Entries are dict objects extracted from the sitemap document. + Usually, the key is the tag name and the value is the text inside it. + + It's important to notice that: + + - as the loc attribute is required, entries without this tag are discarded + - alternate links are stored in a list with the key ``alternate`` + (see ``sitemap_alternate_links``) + - namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname`` + + If you omit this method, all entries found in sitemaps will be + processed, observing other attributes and their settings. + SitemapSpider examples ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 0ee8ba5e7..534c45c70 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -31,6 +31,14 @@ class SitemapSpider(Spider): for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) + def sitemap_filter(self, entries): + """This method can be used to filter sitemap entries by their + attributes, for example, you can filter locs with lastmod greater + than a given date (see docs). + """ + for entry in entries: + yield entry + def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): for url in sitemap_urls_from_robots(response.text, base_url=response.url): @@ -43,12 +51,14 @@ class SitemapSpider(Spider): return s = Sitemap(body) + it = self.sitemap_filter(s) + if s.type == 'sitemapindex': - for loc in iterloc(s, self.sitemap_alternate_links): + for loc in iterloc(it, self.sitemap_alternate_links): if any(x.search(loc) for x in self._follow): yield Request(loc, callback=self._parse_sitemap) elif s.type == 'urlset': - for loc in iterloc(s, self.sitemap_alternate_links): + for loc in iterloc(it, self.sitemap_alternate_links): for r, c in self._cbs: if r.search(loc): yield Request(loc, callback=c) diff --git a/tests/test_spider.py b/tests/test_spider.py index f26da2334..fefdaa403 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -375,6 +375,104 @@ Sitemap: /sitemap-relative-url.xml 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/italiano/']) + def test_sitemap_filter(self): + sitemap = b""" + + + http://www.example.com/english/ + 2010-01-01 + + + http://www.example.com/portuguese/ + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + from datetime import datetime + for entry in entries: + date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') + if date_time.year > 2008: + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/', + 'http://www.example.com/portuguese/']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/']) + + def test_sitemap_filter_with_alternate_links(self): + sitemap = b""" + + + http://www.example.com/english/article_1/ + 2010-01-01 + + + + http://www.example.com/english/article_2/ + 2015-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + alternate_links = entry.get('alternate', tuple()) + for link in alternate_links: + if '/deutsch/' in link: + entry['loc'] = link + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/article_1/', + 'http://www.example.com/english/article_2/']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/deutsch/article_1/']) + + def test_sitemapindex_filter(self): + sitemap = b""" + + + http://www.example.com/sitemap1.xml + 2004-01-01T20:00:00+00:00 + + + http://www.example.com/sitemap2.xml + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + from datetime import datetime + for entry in entries: + date_time = datetime.strptime(entry['lastmod'].split('T')[0], '%Y-%m-%d') + if date_time.year > 2004: + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/sitemap1.xml', + 'http://www.example.com/sitemap2.xml']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/sitemap2.xml']) + class DeprecationTest(unittest.TestCase):