From c68c478ac39b67930758dc4c1d23cae362b8675a Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Sun, 5 Oct 2008 07:45:03 +0000
Subject: [PATCH] added scrapy.contrib.spiders module

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40302
---
 scrapy/trunk/scrapy/contrib/spiders.py | 60 ++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 scrapy/trunk/scrapy/contrib/spiders.py

diff --git a/scrapy/trunk/scrapy/contrib/spiders.py b/scrapy/trunk/scrapy/contrib/spiders.py
new file mode 100644
index 000000000..e03a86cde
--- /dev/null
+++ b/scrapy/trunk/scrapy/contrib/spiders.py
@@ -0,0 +1,60 @@
+"""
+This module contains BasicSpider, a spider class which provides support for
+basic crawling.
+"""
+
+from scrapy.http import Request
+from scrapy.spider import BaseSpider
+from scrapy.core.exceptions import UsageError
+
+class BasicSpider(BaseSpider):
+    """BasicSpider extends BaseSpider by providing support for simple crawling
+    by following links contained in web pages. 
+    
+    With BasicSpider you can write a basic spider very easily and quickly. For
+    more information refer to the Scrapy tutorial"""
+
+    def __init__(self):
+        super(BaseSpider, self).__init__()
+        
+        self._links_callback = []
+
+        for attr in dir(self):
+            if attr.startswith('links_'):
+                suffix = attr.split('_', 1)[1]
+                value = getattr(self, attr)
+                try:
+                    callback = getattr(self, 'parse_%s' % suffix)
+                except AttributeError:
+                    raise UsageError("%s defines links_%s but doesn't provide a parse_%s method" % \
+                                     (type(self).__name__, suffix, suffix))
+                self._links_callback.append((value, callback))
+
+    def parse(self, response):
+        """This function is called by the core for all the start_urls. Do not
+        override this function, override parse_start_url instead."""
+        return self._parse_wrapper(response, self.parse_start_url)
+
+    def parse_start_url(self, response):
+        """Callback function for processing start_urls. It must return a list
+        of ScrapedItems and/or Requests."""
+        return []
+
+    def _links_to_follow(self, response):
+        res = []
+        links_to_follow = {}
+        for lx, callback in self._links_callback:
+            for url, link_text in lx.extract_urls(response).iteritems():
+                links_to_follow[url] = (callback, link_text)
+
+        for url, cb_link in links_to_follow.iteritems():
+            request = Request(url=url, link_text=link_text)
+            request.append_callback(self._parse_wrapper, callback)
+            res.append(request)
+        return res
+
+    def _parse_wrapper(self, response, callback):
+        res = self._links_to_follow(response)
+        res += callback(response) or ()
+        return res
+