From a5e1b7bb4724bafa26b476a87a9f12b4d6479661 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 29 Nov 2018 18:19:14 -0300 Subject: [PATCH 01/11] add sitemap_filter attribute to SitemapSpider class it makes it possible to filter sitemap urls by any available attribute for example, you can filter urls with lastmod greater than a given datetime it can be helpful when the url loc itself does not aggregate that information --- docs/topics/spiders.rst | 26 ++++++++++++++++++++++++++ scrapy/spiders/sitemap.py | 10 ++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index a08dc30f2..b0b9e0483 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -680,6 +680,32 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. + .. attribute:: sitemap_filter + + Specifies a function to filter sitemap entries and their attributes. + + For example:: + + + http://example.com/ + 2005-01-01 + + + We can define a ``sitemap_filter`` function to filter ``urls`` by date:: + + def sitemap_filter(urls): + from datetime import datetime + for url in urls: + date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + if date_time.year >= 2005: + yield url + + This would retrieve only ``urls`` modified on 2005 and the following + years. + + If you omit this attribute, all urls found in sitemaps will be + processed, observing other attributes and their settings. + SitemapSpider examples ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 0ee8ba5e7..907aba243 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -17,6 +17,7 @@ class SitemapSpider(Spider): sitemap_rules = [('', 'parse')] sitemap_follow = [''] sitemap_alternate_links = False + sitemap_filter = None def __init__(self, *a, **kw): super(SitemapSpider, self).__init__(*a, **kw) @@ -43,12 +44,17 @@ class SitemapSpider(Spider): return s = Sitemap(body) + if callable(self.sitemap_filter): + it = self.sitemap_filter(s) + else: + it = s + if s.type == 'sitemapindex': - for loc in iterloc(s, self.sitemap_alternate_links): + for loc in iterloc(it, self.sitemap_alternate_links): if any(x.search(loc) for x in self._follow): yield Request(loc, callback=self._parse_sitemap) elif s.type == 'urlset': - for loc in iterloc(s, self.sitemap_alternate_links): + for loc in iterloc(it, self.sitemap_alternate_links): for r, c in self._cbs: if r.search(loc): yield Request(loc, callback=c) From 672385a371453c84faa2f31425e3701b25260629 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 29 Nov 2018 18:33:20 -0300 Subject: [PATCH 02/11] using a method definition instead of a None attribute --- docs/topics/spiders.rst | 4 ++-- scrapy/spiders/sitemap.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index b0b9e0483..127c8d03e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -680,7 +680,7 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. - .. attribute:: sitemap_filter + .. method:: sitemap_filter(urls) Specifies a function to filter sitemap entries and their attributes. @@ -703,7 +703,7 @@ SitemapSpider This would retrieve only ``urls`` modified on 2005 and the following years. - If you omit this attribute, all urls found in sitemaps will be + If you omit this method, all urls found in sitemaps will be processed, observing other attributes and their settings. diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 907aba243..c86e986db 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -17,7 +17,6 @@ class SitemapSpider(Spider): sitemap_rules = [('', 'parse')] sitemap_follow = [''] sitemap_alternate_links = False - sitemap_filter = None def __init__(self, *a, **kw): super(SitemapSpider, self).__init__(*a, **kw) @@ -32,6 +31,14 @@ class SitemapSpider(Spider): for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) + def sitemap_filter(self, urls): + """This method can be used to filter sitemap entries by their + attributes, for example, you can filter locs with lastmod greater + than a given date (see docs). + """ + for url in urls: + yield url + def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): for url in sitemap_urls_from_robots(response.text, base_url=response.url): @@ -44,10 +51,7 @@ class SitemapSpider(Spider): return s = Sitemap(body) - if callable(self.sitemap_filter): - it = self.sitemap_filter(s) - else: - it = s + it = self.sitemap_filter(s) if s.type == 'sitemapindex': for loc in iterloc(it, self.sitemap_alternate_links): From d7d5917ff12ecb8db7cd04592f7cc18b0ab1a996 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 30 Nov 2018 11:20:12 -0300 Subject: [PATCH 03/11] add tests for the sitemap_filter method in the SitemapSpider class --- tests/test_spider.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index f26da2334..871852ab2 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -375,6 +375,38 @@ Sitemap: /sitemap-relative-url.xml 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/italiano/']) + def test_sitemap_filter(self): + sitemap = b""" + + + http://www.example.com/english/ + 2010-01-01 + + + http://www.example.com/portuguese/ + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, urls): + from datetime import datetime + for url in urls: + date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + if date_time.year > 2008: + yield url + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/', + 'http://www.example.com/portuguese/']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/']) + class DeprecationTest(unittest.TestCase): From 657f0663b3cb97ca1c1a498c066de444bd30fa82 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 20 Dec 2018 13:35:52 -0300 Subject: [PATCH 04/11] rename param from urls to entries --- docs/topics/spiders.rst | 16 ++++++++-------- scrapy/spiders/sitemap.py | 6 +++--- tests/test_spider.py | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 127c8d03e..918f1cc36 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -680,7 +680,7 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. - .. method:: sitemap_filter(urls) + .. method:: sitemap_filter(entries) Specifies a function to filter sitemap entries and their attributes. @@ -691,19 +691,19 @@ SitemapSpider 2005-01-01 - We can define a ``sitemap_filter`` function to filter ``urls`` by date:: + We can define a ``sitemap_filter`` function to filter ``entries`` by date:: - def sitemap_filter(urls): + def sitemap_filter(entries): from datetime import datetime - for url in urls: - date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + for entry in entries: + date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') if date_time.year >= 2005: - yield url + yield entry - This would retrieve only ``urls`` modified on 2005 and the following + This would retrieve only ``entries`` modified on 2005 and the following years. - If you omit this method, all urls found in sitemaps will be + If you omit this method, all entries found in sitemaps will be processed, observing other attributes and their settings. diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index c86e986db..534c45c70 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -31,13 +31,13 @@ class SitemapSpider(Spider): for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) - def sitemap_filter(self, urls): + def sitemap_filter(self, entries): """This method can be used to filter sitemap entries by their attributes, for example, you can filter locs with lastmod greater than a given date (see docs). """ - for url in urls: - yield url + for entry in entries: + yield entry def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): diff --git a/tests/test_spider.py b/tests/test_spider.py index 871852ab2..d5d10c9ea 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -390,12 +390,12 @@ Sitemap: /sitemap-relative-url.xml """ class FilteredSitemapSpider(self.spider_class): - def sitemap_filter(self, urls): + def sitemap_filter(self, entries): from datetime import datetime - for url in urls: - date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + for entry in entries: + date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') if date_time.year > 2008: - yield url + yield entry r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") From 5e7ecf9dc1954060fd0445dce5fb54e020dd3e59 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 21 Dec 2018 17:31:52 -0300 Subject: [PATCH 05/11] add tests for sitemapindex --- tests/test_spider.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index d5d10c9ea..8b56cfec1 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -407,6 +407,37 @@ Sitemap: /sitemap-relative-url.xml self.assertEqual([req.url for req in spider._parse_sitemap(r)], ['http://www.example.com/english/']) + def test_sitemapindex_filter(self): + sitemap = b""" + + + http://www.example.com/sitemap1.xml + 2004-01-01T20:00:00+00:00 + + + http://www.example.com/sitemap2.xml + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + from datetime import datetime + for entry in entries: + date_time = datetime.strptime(entry['lastmod'].split('T')[0], '%Y-%m-%d') + if date_time.year > 2004: + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/sitemap1.xml', + 'http://www.example.com/sitemap2.xml']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/sitemap2.xml']) + class DeprecationTest(unittest.TestCase): From 10f46bca54b2879da02641159e53453fe0cc97dc Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 26 Dec 2018 11:20:18 -0300 Subject: [PATCH 06/11] documenting sitemap entries as suggested by @kmike --- docs/topics/spiders.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 918f1cc36..9d4ed6ca6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -703,6 +703,16 @@ SitemapSpider This would retrieve only ``entries`` modified on 2005 and the following years. + Entries are dict objects extracted from the sitemap document. + Usually, the key is the tag name and the value is the text inside it. + + It's important to notice that: + + - as the loc attribute is required, entries without this tag are discarded + - alternate links are stored in a list with the key ``alternate`` + (see ``sitemap_alternate_links``) + - namespaces are removed, so lxml tags named as ``{foo}bar`` become only ``bar`` + If you omit this method, all entries found in sitemaps will be processed, observing other attributes and their settings. From fe283bcd058734f88977a2033dfa36664e7ee619 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 26 Dec 2018 12:32:22 -0300 Subject: [PATCH 07/11] add test case for sitemap filter with alternate links --- tests/test_spider.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index 8b56cfec1..fefdaa403 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -407,6 +407,41 @@ Sitemap: /sitemap-relative-url.xml self.assertEqual([req.url for req in spider._parse_sitemap(r)], ['http://www.example.com/english/']) + def test_sitemap_filter_with_alternate_links(self): + sitemap = b""" + + + http://www.example.com/english/article_1/ + 2010-01-01 + + + + http://www.example.com/english/article_2/ + 2015-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + alternate_links = entry.get('alternate', tuple()) + for link in alternate_links: + if '/deutsch/' in link: + entry['loc'] = link + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/article_1/', + 'http://www.example.com/english/article_2/']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/deutsch/article_1/']) + def test_sitemapindex_filter(self): sitemap = b""" From e1597f7c420ead9a563677aab61f18f9b89640a9 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 26 Dec 2018 15:05:21 -0300 Subject: [PATCH 08/11] improve readability --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 9d4ed6ca6..c47a2fca0 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -711,7 +711,7 @@ SitemapSpider - as the loc attribute is required, entries without this tag are discarded - alternate links are stored in a list with the key ``alternate`` (see ``sitemap_alternate_links``) - - namespaces are removed, so lxml tags named as ``{foo}bar`` become only ``bar`` + - namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname`` If you omit this method, all entries found in sitemaps will be processed, observing other attributes and their settings. From b68308779a6d2ce7deda3675d0bcdf671a4fb935 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 27 Dec 2018 17:37:59 -0300 Subject: [PATCH 09/11] improving docs --- docs/topics/spiders.rst | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index c47a2fca0..4f7135309 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -682,7 +682,8 @@ SitemapSpider .. method:: sitemap_filter(entries) - Specifies a function to filter sitemap entries and their attributes. + This is a filter funtion that could be overridden to select sitemap entries + based on their attributes. For example:: @@ -693,12 +694,17 @@ SitemapSpider We can define a ``sitemap_filter`` function to filter ``entries`` by date:: - def sitemap_filter(entries): - from datetime import datetime - for entry in entries: - date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') - if date_time.year >= 2005: - yield entry + class FilteredSitemapSpider(scrapy.SitemapSpider): + name = 'filtered_sitemap_spider' + allowed_domains = ['example.com'] + sitemap_urls = ['http://example.com/sitemap.xml'] + + def sitemap_filter(self, entries): + from datetime import datetime + for entry in entries: + date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') + if date_time.year >= 2005: + yield entry This would retrieve only ``entries`` modified on 2005 and the following years. From bfbcf52e9df77af7a7c9a8a7a711e06612be4763 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 27 Dec 2018 18:12:31 -0300 Subject: [PATCH 10/11] fix SitemapSpider import --- docs/topics/spiders.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 4f7135309..39410d66e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -694,7 +694,9 @@ SitemapSpider We can define a ``sitemap_filter`` function to filter ``entries`` by date:: - class FilteredSitemapSpider(scrapy.SitemapSpider): + from scrapy.spiders.sitemap import SitemapSpider + + class FilteredSitemapSpider(SitemapSpider): name = 'filtered_sitemap_spider' allowed_domains = ['example.com'] sitemap_urls = ['http://example.com/sitemap.xml'] From 5a824c906c501a204624ea7b4fb99904807c8b81 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 27 Dec 2018 18:34:41 -0300 Subject: [PATCH 11/11] using shorter import version and moving datetime import to the beginning of the code snippet --- docs/topics/spiders.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 39410d66e..742a88659 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -694,7 +694,8 @@ SitemapSpider We can define a ``sitemap_filter`` function to filter ``entries`` by date:: - from scrapy.spiders.sitemap import SitemapSpider + from datetime import datetime + from scrapy.spiders import SitemapSpider class FilteredSitemapSpider(SitemapSpider): name = 'filtered_sitemap_spider' @@ -702,7 +703,6 @@ SitemapSpider sitemap_urls = ['http://example.com/sitemap.xml'] def sitemap_filter(self, entries): - from datetime import datetime for entry in entries: date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') if date_time.year >= 2005: