mirror of https://github.com/scrapy/scrapy.git
rename param from urls to entries
This commit is contained in:
parent
d7d5917ff1
commit
657f0663b3
|
|
@ -680,7 +680,7 @@ SitemapSpider
|
|||
|
||||
Default is ``sitemap_alternate_links`` disabled.
|
||||
|
||||
.. method:: sitemap_filter(urls)
|
||||
.. method:: sitemap_filter(entries)
|
||||
|
||||
Specifies a function to filter sitemap entries and their attributes.
|
||||
|
||||
|
|
@ -691,19 +691,19 @@ SitemapSpider
|
|||
<lastmod>2005-01-01</lastmod>
|
||||
</url>
|
||||
|
||||
We can define a ``sitemap_filter`` function to filter ``urls`` by date::
|
||||
We can define a ``sitemap_filter`` function to filter ``entries`` by date::
|
||||
|
||||
def sitemap_filter(urls):
|
||||
def sitemap_filter(entries):
|
||||
from datetime import datetime
|
||||
for url in urls:
|
||||
date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')
|
||||
for entry in entries:
|
||||
date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
|
||||
if date_time.year >= 2005:
|
||||
yield url
|
||||
yield entry
|
||||
|
||||
This would retrieve only ``urls`` modified on 2005 and the following
|
||||
This would retrieve only ``entries`` modified on 2005 and the following
|
||||
years.
|
||||
|
||||
If you omit this method, all urls found in sitemaps will be
|
||||
If you omit this method, all entries found in sitemaps will be
|
||||
processed, observing other attributes and their settings.
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -31,13 +31,13 @@ class SitemapSpider(Spider):
|
|||
for url in self.sitemap_urls:
|
||||
yield Request(url, self._parse_sitemap)
|
||||
|
||||
def sitemap_filter(self, urls):
|
||||
def sitemap_filter(self, entries):
|
||||
"""This method can be used to filter sitemap entries by their
|
||||
attributes, for example, you can filter locs with lastmod greater
|
||||
than a given date (see docs).
|
||||
"""
|
||||
for url in urls:
|
||||
yield url
|
||||
for entry in entries:
|
||||
yield entry
|
||||
|
||||
def _parse_sitemap(self, response):
|
||||
if response.url.endswith('/robots.txt'):
|
||||
|
|
|
|||
|
|
@ -390,12 +390,12 @@ Sitemap: /sitemap-relative-url.xml
|
|||
</urlset>"""
|
||||
|
||||
class FilteredSitemapSpider(self.spider_class):
|
||||
def sitemap_filter(self, urls):
|
||||
def sitemap_filter(self, entries):
|
||||
from datetime import datetime
|
||||
for url in urls:
|
||||
date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')
|
||||
for entry in entries:
|
||||
date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
|
||||
if date_time.year > 2008:
|
||||
yield url
|
||||
yield entry
|
||||
|
||||
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
|
||||
spider = self.spider_class("example.com")
|
||||
|
|
|
|||
Loading…
Reference in New Issue