rename param from urls to entries

This commit is contained in:
Victor Torres 2018-12-20 13:35:52 -03:00
parent d7d5917ff1
commit 657f0663b3
3 changed files with 15 additions and 15 deletions

View File

@ -680,7 +680,7 @@ SitemapSpider
Default is ``sitemap_alternate_links`` disabled.
.. method:: sitemap_filter(urls)
.. method:: sitemap_filter(entries)
Specifies a function to filter sitemap entries and their attributes.
@ -691,19 +691,19 @@ SitemapSpider
<lastmod>2005-01-01</lastmod>
</url>
We can define a ``sitemap_filter`` function to filter ``urls`` by date::
We can define a ``sitemap_filter`` function to filter ``entries`` by date::
def sitemap_filter(urls):
def sitemap_filter(entries):
from datetime import datetime
for url in urls:
date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')
for entry in entries:
date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
if date_time.year >= 2005:
yield url
yield entry
This would retrieve only ``urls`` modified on 2005 and the following
This would retrieve only ``entries`` modified on 2005 and the following
years.
If you omit this method, all urls found in sitemaps will be
If you omit this method, all entries found in sitemaps will be
processed, observing other attributes and their settings.

View File

@ -31,13 +31,13 @@ class SitemapSpider(Spider):
for url in self.sitemap_urls:
yield Request(url, self._parse_sitemap)
def sitemap_filter(self, urls):
def sitemap_filter(self, entries):
"""This method can be used to filter sitemap entries by their
attributes, for example, you can filter locs with lastmod greater
than a given date (see docs).
"""
for url in urls:
yield url
for entry in entries:
yield entry
def _parse_sitemap(self, response):
if response.url.endswith('/robots.txt'):

View File

@ -390,12 +390,12 @@ Sitemap: /sitemap-relative-url.xml
</urlset>"""
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, urls):
def sitemap_filter(self, entries):
from datetime import datetime
for url in urls:
date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')
for entry in entries:
date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
if date_time.year > 2008:
yield url
yield entry
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")