From d7d5917ff12ecb8db7cd04592f7cc18b0ab1a996 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Fri, 30 Nov 2018 11:20:12 -0300
Subject: [PATCH] add tests for the sitemap_filter method in the SitemapSpider
 class

---
 tests/test_spider.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/tests/test_spider.py b/tests/test_spider.py
index f26da2334..871852ab2 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -375,6 +375,38 @@ Sitemap: /sitemap-relative-url.xml
                           'http://www.example.com/schweiz-deutsch/',
                           'http://www.example.com/italiano/'])
 
+    def test_sitemap_filter(self):
+        sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:xhtml="http://www.w3.org/1999/xhtml">
+        <url>
+            <loc>http://www.example.com/english/</loc>
+            <lastmod>2010-01-01</lastmod>
+        </url>
+        <url>
+            <loc>http://www.example.com/portuguese/</loc>
+            <lastmod>2005-01-01</lastmod>
+        </url>
+    </urlset>"""
+
+        class FilteredSitemapSpider(self.spider_class):
+            def sitemap_filter(self, urls):
+                from datetime import datetime
+                for url in urls:
+                    date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')
+                    if date_time.year > 2008:
+                        yield url
+
+        r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
+        spider = self.spider_class("example.com")
+        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
+                         ['http://www.example.com/english/',
+                          'http://www.example.com/portuguese/'])
+
+        spider = FilteredSitemapSpider("example.com")
+        self.assertEqual([req.url for req in spider._parse_sitemap(r)],
+                         ['http://www.example.com/english/'])
+
 
 class DeprecationTest(unittest.TestCase):