From 57c43fdce6809382b360e1dff61b1289d62ce679 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 15 Jun 2011 11:54:34 -0300 Subject: [PATCH] added SitemapSpider, with tests and doc --- docs/intro/overview.rst | 3 + docs/topics/spiders.rst | 124 ++++++++++++++++++++++++++++- scrapy/contrib/spiders/__init__.py | 1 + scrapy/contrib/spiders/sitemap.py | 41 ++++++++++ scrapy/tests/test_utils_sitemap.py | 65 +++++++++++++++ scrapy/utils/sitemap.py | 35 ++++++++ 6 files changed, 266 insertions(+), 3 deletions(-) create mode 100644 scrapy/contrib/spiders/sitemap.py create mode 100644 scrapy/tests/test_utils_sitemap.py create mode 100644 scrapy/utils/sitemap.py diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 080b3980d..25dd4c11d 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -246,6 +246,8 @@ scraping easy and efficient, such as: * :ref:`Logging ` facility that you can hook on to for catching errors during the scraping process. +* Support for crawling based on URLs discovered through `Sitemaps`_ + What's next? ============ @@ -262,3 +264,4 @@ interest! .. _XPath: http://www.w3.org/TR/xpath .. _XPath reference: http://www.w3.org/TR/xpath .. _Amazon S3: http://aws.amazon.com/s3/ +.. _Sitemaps: http://www.sitemaps.org diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index dd5f6a6e2..83ce12e24 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -45,8 +45,13 @@ We will talk about those types here. Built-in spiders reference ========================== -For the examples used in the following spiders reference, we'll assume we have a -``TestItem`` declared in a ``myproject.items`` module, in your project:: +Scrapy comes with some useful generic spiders that you can use, to subclass +your spiders from. Their aim is to provide convenient functionality for a few +common scraping cases, like following all links on a site based on certain +rules, crawling from `Sitemaps`_, or parsing a XML/CSV feed. + +For the examples used in the following spiders, we'll assume you have a project +with a ``TestItem`` declared in a ``myproject.items`` module:: from scrapy.item import Item @@ -228,6 +233,7 @@ CrawlSpider Crawling rules ~~~~~~~~~~~~~~ + .. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None) ``link_extractor`` is a :ref:`Link Extractor ` object which @@ -262,7 +268,7 @@ Crawling rules filter out the request). CrawlSpider example -------------------- +~~~~~~~~~~~~~~~~~~~ Let's now take a look at an example CrawlSpider with rules:: @@ -465,3 +471,115 @@ Let's see an example similar to the previous one, but using a item['name'] = row['name'] item['description'] = row['description'] return item + + +SitemapSpider +------------- + +.. class:: SitemapSpider + + SitemapSpider allows you to crawl a site by discovering the URLs using + `Sitemaps`_. + + It supports nested sitemaps and discovering sitemap urls from + `robots.txt`_. + + .. attribute:: sitemap_urls + + A list of urls pointing to the sitemaps whose urls you want to crawl. + + You can also point to a `robots.txt`_ and it will be parsed to extract + sitemap urls from it. + + .. attribute:: sitemap_rules + + A list of tuples ``(regex, callback)`` where: + + * ``regex`` is a regular expression to match urls extracted from sitemaps. + ``regex`` can be either a str or a compiled regex object. + + * callback is the callback to use for processing the urls that match + the regular expression. ``callback`` can be a string (indicating the + name of a spider method) or a callable. + + For example:: + + sitemap_rules = [('/product/', 'parse_product')] + + Rules are applied in order, and only the first one that matches will be + used. + + If you omit this attribute, all urls found in sitemaps will be + processed with the ``parse`` callback. + +SitemapSpider examples +~~~~~~~~~~~~~~~~~~~~~~ + +Simplest example: process all urls discovered through sitemaps using the +``parse`` callback:: + + from scrapy.contrib.spiders import SitemapSpider + + class MySpider(SitemapSpider): + sitemap_urls = ['http://www.example.com/sitemap.xml'] + + def parse(self, response): + pass # ... scrape item here ... + +Process some urls with certain callback and other urls with a different +callback:: + + from scrapy.contrib.spiders import SitemapSpider + + class MySpider(SitemapSpider): + sitemap_urls = ['http://www.example.com/sitemap.xml'] + sitemap_rules = [ + ('/product/', 'parse_product'), + ('/category/', 'parse_category'), + ] + + def parse_product(self, response): + pass # ... scrape product ... + + def parse_category(self, response): + pass # ... scrape category ... + +Follow sitemaps defined in the `robots.txt`_ file:: + + from scrapy.contrib.spiders import SitemapSpider + + class MySpider(SitemapSpider): + sitemap_urls = ['http://www.example.com/robots.txt'] + sitemap_rules = [ + ('/shop/', 'parse_shop'), + ] + + def parse_shop(self, response): + pass # ... scrape shop here ... + +Combine SitemapSpider with other sources of urls:: + + from scrapy.contrib.spiders import SitemapSpider + + class MySpider(SitemapSpider): + sitemap_urls = ['http://www.example.com/robots.txt'] + sitemap_rules = [ + ('/shop/', 'parse_shop'), + ] + + other_urls = ['http://www.example.com/about'] + + def start_requests(self): + requests = list(super(MySpider, self).start_requests()) + requests += [Request(x, callback=self.parse_other) for x in self.other_urls] + return requests + + def parse_shop(self, response): + pass # ... scrape shop here ... + + def parse_other(self, response): + pass # ... scrape other here ... + +.. _Sitemaps: http://www.sitemaps.org +.. _robots.txt: http://www.robotstxt.org/ + diff --git a/scrapy/contrib/spiders/__init__.py b/scrapy/contrib/spiders/__init__.py index 6a310dd44..c16bb6c0a 100644 --- a/scrapy/contrib/spiders/__init__.py +++ b/scrapy/contrib/spiders/__init__.py @@ -1,2 +1,3 @@ from scrapy.contrib.spiders.crawl import CrawlSpider, Rule from scrapy.contrib.spiders.feed import XMLFeedSpider, CSVFeedSpider +from scrapy.contrib.spiders.sitemap import SitemapSpider diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py new file mode 100644 index 000000000..0aaf96e44 --- /dev/null +++ b/scrapy/contrib/spiders/sitemap.py @@ -0,0 +1,41 @@ +import re + +from scrapy.spider import BaseSpider +from scrapy.http import Request +from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots + +class SitemapSpider(BaseSpider): + + sitemap_urls = () + sitemap_rules = [('', 'parse')] + + def __init__(self, *a, **kw): + super(SitemapSpider, self).__init__(*a, **kw) + self._cbs = [] + for r, c in self.sitemap_rules: + if isinstance(r, basestring): + r = re.compile(r) + if isinstance(c, basestring): + c = getattr(self, c) + self._cbs.append((r, c)) + print self._cbs + + def start_requests(self): + return [Request(x, callback=self._parse_sitemap) for x in self.sitemap_urls] + + def _parse_sitemap(self, response): + if response.url.endswith('/robots.txt'): + for url in sitemap_urls_from_robots(response.body): + yield Request(url, callback=self._parse_sitemap) + else: + s = Sitemap(response.body) + if s.type == 'sitemapindex': + for sitemap in s: + yield Request(sitemap['loc'], callback=self._parse_sitemap) + elif s.type == 'urlset': + for url in s: + loc = url['loc'] + for r, c in self._cbs: + if r.search(loc): + yield Request(loc, callback=c) + break diff --git a/scrapy/tests/test_utils_sitemap.py b/scrapy/tests/test_utils_sitemap.py new file mode 100644 index 000000000..d78447ef0 --- /dev/null +++ b/scrapy/tests/test_utils_sitemap.py @@ -0,0 +1,65 @@ +import unittest + +from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots + +class SitemapTest(unittest.TestCase): + + def test_sitemap(self): + s = Sitemap(""" + + + http://www.example.com/ + 2009-08-16 + daily + 1 + + + http://www.example.com/Special-Offers.html + 2009-08-16 + weekly + 0.8 + +""") + assert s.type == 'urlset' + self.assertEqual(list(s), + [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}]) + + def test_sitemap_index(self): + s = Sitemap(""" + + + http://www.example.com/sitemap1.xml.gz + 2004-10-01T18:23:17+00:00 + + + http://www.example.com/sitemap2.xml.gz + 2005-01-01 + +""") + assert s.type == 'sitemapindex' + self.assertEqual(list(s), [{'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'}, {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'}]) + +class RobotsTest(unittest.TestCase): + + def test_sitemap_urls_from_robots(self): + robots = """User-agent: * +Disallow: /aff/ +Disallow: /wl/ + +# Search and shopping refining +Disallow: /s*/*facet +Disallow: /s*/*tags + +# Sitemap files +Sitemap: http://example.com/sitemap.xml +Sitemap: http://example.com/sitemap-product-index.xml + +# Forums +Disallow: /forum/search/ +Disallow: /forum/active/ +""" + self.assertEqual(list(sitemap_urls_from_robots(robots)), + ['http://example.com/sitemap.xml', 'http://example.com/sitemap-product-index.xml']) + +if __name__ == '__main__': + unittest.main() diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py new file mode 100644 index 000000000..aad39c512 --- /dev/null +++ b/scrapy/utils/sitemap.py @@ -0,0 +1,35 @@ +""" +Module for processing Sitemaps. + +Note: The main purpose of this module is to provide support for the +SitemapSpider, its API is subject to change without notice. +""" + +from cStringIO import StringIO +from xml.etree.cElementTree import ElementTree + +class Sitemap(object): + """Class to parse Sitemap (type=urlset) and Sitemap Index + (type=sitemapindex) files""" + + def __init__(self, xmltext): + tree = ElementTree() + tree.parse(StringIO(xmltext)) + self._root = tree.getroot() + _, self.type = self._root.tag.split('}', 1) + + def __iter__(self): + for elem in self._root.getchildren(): + d = {} + for el in elem.getchildren(): + _, name = el.tag.split('}', 1) + d[name] = el.text + yield d + +def sitemap_urls_from_robots(robots_text): + """Return an iterator over all sitemap urls contained in the given + robots.txt file + """ + for line in robots_text.splitlines(): + if line.lstrip().startswith('Sitemap:'): + yield line.split(':', 1)[1].strip()