added SitemapSpider, with tests and doc

docs/intro/overview.rst

@@ -246,6 +246,8 @@ scraping easy and efficient, such as:
 * :ref:`Logging <topics-logging>` facility that you can hook on to for catching
   errors during the scraping process.
 
+* Support for crawling based on URLs discovered through `Sitemaps`_
+
 What's next?
 ============
 
@@ -262,3 +264,4 @@ interest!
 .. _XPath: http://www.w3.org/TR/xpath
 .. _XPath reference: http://www.w3.org/TR/xpath
 .. _Amazon S3: http://aws.amazon.com/s3/
+.. _Sitemaps: http://www.sitemaps.org

docs/topics/spiders.rst

@@ -45,8 +45,13 @@ We will talk about those types here.
 Built-in spiders reference
 ==========================
 
-For the examples used in the following spiders reference, we'll assume we have a
-``TestItem`` declared in a ``myproject.items`` module, in your project::
+Scrapy comes with some useful generic spiders that you can use, to subclass
+your spiders from. Their aim is to provide convenient functionality for a few
+common scraping cases, like following all links on a site based on certain
+rules, crawling from `Sitemaps`_, or parsing a XML/CSV feed.
+
+For the examples used in the following spiders, we'll assume you have a project
+with a ``TestItem`` declared in a ``myproject.items`` module::
 
     from scrapy.item import Item
 
@@ -228,6 +233,7 @@ CrawlSpider
        
 Crawling rules
 ~~~~~~~~~~~~~~
+
 .. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None)
 
    ``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
@@ -262,7 +268,7 @@ Crawling rules
     filter out the request).
 
 CrawlSpider example
--------------------
+~~~~~~~~~~~~~~~~~~~
 
 Let's now take a look at an example CrawlSpider with rules::
 
@@ -465,3 +471,115 @@ Let's see an example similar to the previous one, but using a
             item['name'] = row['name']
             item['description'] = row['description']
             return item
+
+
+SitemapSpider
+-------------
+
+.. class:: SitemapSpider
+
+    SitemapSpider allows you to crawl a site by discovering the URLs using
+    `Sitemaps`_.
+
+    It supports nested sitemaps and discovering sitemap urls from
+    `robots.txt`_.
+
+    .. attribute:: sitemap_urls
+
+        A list of urls pointing to the sitemaps whose urls you want to crawl.
+
+        You can also point to a `robots.txt`_ and it will be parsed to extract
+        sitemap urls from it.
+
+    .. attribute:: sitemap_rules
+
+        A list of tuples ``(regex, callback)`` where:
+
+        * ``regex`` is a regular expression to match urls extracted from sitemaps.
+          ``regex`` can be either a str or a compiled regex object.
+
+        * callback is the callback to use for processing the urls that match
+          the regular expression. ``callback`` can be a string (indicating the
+          name of a spider method) or a callable.
+
+        For example::
+
+            sitemap_rules = [('/product/', 'parse_product')]
+
+        Rules are applied in order, and only the first one that matches will be
+        used.
+
+        If you omit this attribute, all urls found in sitemaps will be
+        processed with the ``parse`` callback.
+
+SitemapSpider examples
+~~~~~~~~~~~~~~~~~~~~~~
+
+Simplest example: process all urls discovered through sitemaps using the
+``parse`` callback::
+
+    from scrapy.contrib.spiders import SitemapSpider
+
+    class MySpider(SitemapSpider):
+        sitemap_urls = ['http://www.example.com/sitemap.xml']
+
+        def parse(self, response):
+            pass # ... scrape item here ...
+
+Process some urls with certain callback and other urls with a different
+callback::
+
+    from scrapy.contrib.spiders import SitemapSpider
+
+    class MySpider(SitemapSpider):
+        sitemap_urls = ['http://www.example.com/sitemap.xml']
+        sitemap_rules = [
+            ('/product/', 'parse_product'),
+            ('/category/', 'parse_category'),
+        ]
+
+        def parse_product(self, response):
+            pass # ... scrape product ...
+
+        def parse_category(self, response):
+            pass # ... scrape category ...
+
+Follow sitemaps defined in the `robots.txt`_ file::
+
+    from scrapy.contrib.spiders import SitemapSpider
+
+    class MySpider(SitemapSpider):
+        sitemap_urls = ['http://www.example.com/robots.txt']
+        sitemap_rules = [
+            ('/shop/', 'parse_shop'),
+        ]
+
+        def parse_shop(self, response):
+            pass # ... scrape shop here ...
+
+Combine SitemapSpider with other sources of urls::
+
+    from scrapy.contrib.spiders import SitemapSpider
+
+    class MySpider(SitemapSpider):
+        sitemap_urls = ['http://www.example.com/robots.txt']
+        sitemap_rules = [
+            ('/shop/', 'parse_shop'),
+        ]
+
+        other_urls = ['http://www.example.com/about']
+
+        def start_requests(self):
+            requests = list(super(MySpider, self).start_requests())
+            requests += [Request(x, callback=self.parse_other) for x in self.other_urls]
+            return requests
+
+        def parse_shop(self, response):
+            pass # ... scrape shop here ...
+
+        def parse_other(self, response):
+            pass # ... scrape other here ...
+
+.. _Sitemaps: http://www.sitemaps.org
+.. _robots.txt: http://www.robotstxt.org/
+

scrapy/contrib/spiders/__init__.py

@@ -1,2 +1,3 @@
 from scrapy.contrib.spiders.crawl import CrawlSpider, Rule
 from scrapy.contrib.spiders.feed import XMLFeedSpider, CSVFeedSpider
+from scrapy.contrib.spiders.sitemap import SitemapSpider

scrapy/contrib/spiders/sitemap.py

@@ -0,0 +1,41 @@
+import re
+
+from scrapy.spider import BaseSpider
+from scrapy.http import Request
+from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
+
+class SitemapSpider(BaseSpider):
+
+    sitemap_urls = ()
+    sitemap_rules = [('', 'parse')]
+
+    def __init__(self, *a, **kw):
+        super(SitemapSpider, self).__init__(*a, **kw)
+        self._cbs = []
+        for r, c in self.sitemap_rules:
+            if isinstance(r, basestring):
+                r = re.compile(r)
+            if isinstance(c, basestring):
+                c = getattr(self, c)
+            self._cbs.append((r, c))
+            print self._cbs
+
+    def start_requests(self):
+        return [Request(x, callback=self._parse_sitemap) for x in self.sitemap_urls]
+
+    def _parse_sitemap(self, response):
+        if response.url.endswith('/robots.txt'):
+            for url in sitemap_urls_from_robots(response.body):
+                yield Request(url, callback=self._parse_sitemap)
+        else:
+            s = Sitemap(response.body)
+            if s.type == 'sitemapindex':
+                for sitemap in s:
+                    yield Request(sitemap['loc'], callback=self._parse_sitemap)
+            elif s.type == 'urlset':
+                for url in s:
+                    loc = url['loc']
+                    for r, c in self._cbs:
+                        if r.search(loc):
+                            yield Request(loc, callback=c)
+                            break

scrapy/tests/test_utils_sitemap.py

@@ -0,0 +1,65 @@
+import unittest
+
+from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
+
+class SitemapTest(unittest.TestCase):
+
+    def test_sitemap(self):
+        s = Sitemap("""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
+  <url>
+    <loc>http://www.example.com/</loc>
+    <lastmod>2009-08-16</lastmod>
+    <changefreq>daily</changefreq>
+    <priority>1</priority>
+  </url>
+  <url>
+    <loc>http://www.example.com/Special-Offers.html</loc>
+    <lastmod>2009-08-16</lastmod>
+    <changefreq>weekly</changefreq>
+    <priority>0.8</priority>
+  </url>
+</urlset>""")
+        assert s.type == 'urlset'
+        self.assertEqual(list(s),
+            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}])
+
+    def test_sitemap_index(self):
+        s = Sitemap("""<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+   <sitemap>
+      <loc>http://www.example.com/sitemap1.xml.gz</loc>
+      <lastmod>2004-10-01T18:23:17+00:00</lastmod>
+   </sitemap>
+   <sitemap>
+      <loc>http://www.example.com/sitemap2.xml.gz</loc>
+      <lastmod>2005-01-01</lastmod>
+   </sitemap>
+</sitemapindex>""")
+        assert s.type == 'sitemapindex'
+        self.assertEqual(list(s), [{'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'}, {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'}])
+
+class RobotsTest(unittest.TestCase):
+
+    def test_sitemap_urls_from_robots(self):
+        robots = """User-agent: *
+Disallow: /aff/
+Disallow: /wl/
+
+# Search and shopping refining
+Disallow: /s*/*facet
+Disallow: /s*/*tags
+
+# Sitemap files
+Sitemap: http://example.com/sitemap.xml
+Sitemap: http://example.com/sitemap-product-index.xml
+
+# Forums 
+Disallow: /forum/search/
+Disallow: /forum/active/
+"""
+        self.assertEqual(list(sitemap_urls_from_robots(robots)), 
+             ['http://example.com/sitemap.xml', 'http://example.com/sitemap-product-index.xml'])
+
+if __name__ == '__main__':
+    unittest.main()

scrapy/utils/sitemap.py

@@ -0,0 +1,35 @@
+"""
+Module for processing Sitemaps.
+
+Note: The main purpose of this module is to provide support for the
+SitemapSpider, its API is subject to change without notice.
+"""
+
+from cStringIO import StringIO
+from xml.etree.cElementTree import ElementTree
+
+class Sitemap(object):
+    """Class to parse Sitemap (type=urlset) and Sitemap Index
+    (type=sitemapindex) files"""
+
+    def __init__(self, xmltext):
+        tree = ElementTree()
+        tree.parse(StringIO(xmltext))
+        self._root = tree.getroot()
+        _, self.type = self._root.tag.split('}', 1)
+
+    def __iter__(self):
+        for elem in self._root.getchildren():
+            d = {}
+            for el in elem.getchildren():
+                _, name = el.tag.split('}', 1)
+                d[name] = el.text
+            yield d
+
+def sitemap_urls_from_robots(robots_text):
+    """Return an iterator over all sitemap urls contained in the given
+    robots.txt file
+    """
+    for line in robots_text.splitlines():
+        if line.lstrip().startswith('Sitemap:'):
+            yield line.split(':', 1)[1].strip()