examples/experimental: added gooledir crawler

examples/experimental/googledir/googledir/__init__.py

@@ -0,0 +1 @@
+# googledir project

examples/experimental/googledir/googledir/items.py

@@ -0,0 +1,16 @@
+# Define here the models for your scraped items
+#
+# See documentation in:
+# http://doc.scrapy.org/topics/items.html
+
+from scrapy.item import Item, Field
+
+class GoogledirItem(Item):
+
+    name = Field(default='')
+    url = Field(default='')
+    description = Field(default='')
+
+    def __str__(self):
+        return "Google Category: name=%s url=%s" \
+                    % (self['name'], self['url'])

examples/experimental/googledir/googledir/pipelines.py

@@ -0,0 +1,22 @@
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: http://doc.scrapy.org/topics/item-pipeline.html
+
+from scrapy.core.exceptions import DropItem
+
+class FilterWordsPipeline(object):
+    """
+    A pipeline for filtering out items which contain certain 
+    words in their description
+    """ 
+
+    # put all words in lowercase
+    words_to_filter = ['politics', 'religion']
+
+    def process_item(self, spider, item):
+        for word in self.words_to_filter:
+            if word in unicode(item['description']).lower():
+                raise DropItem("Contains forbidden word: %s" % word)
+        else:
+            return item

examples/experimental/googledir/googledir/settings.py

@@ -0,0 +1,21 @@
+# Scrapy settings for googledir project
+#
+# For simplicity, this file contains only the most important settings by
+# default. All the other settings are documented here:
+#
+#     http://doc.scrapy.org/topics/settings.html
+#
+# Or you can copy and paste them from where they're defined in Scrapy:
+# 
+#     scrapy/conf/default_settings.py
+#
+
+BOT_NAME = 'googledir'
+BOT_VERSION = '1.0'
+
+SPIDER_MODULES = ['googledir.spiders']
+NEWSPIDER_MODULE = 'googledir.spiders'
+DEFAULT_ITEM_CLASS = 'googledir.items.GoogledirItem'
+USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
+
+ITEM_PIPELINES = ['googledir.pipelines.FilterWordsPipeline']

examples/experimental/googledir/googledir/spiders/__init__.py

@@ -0,0 +1,8 @@
+# This package will contain the spiders of your Scrapy project
+#
+# To create the first spider for your project use this command:
+#
+#   scrapy-ctl.py genspider myspider myspider-domain.com
+#
+# For more info see:
+# http://doc.scrapy.org/topics/spiders.html

examples/experimental/googledir/googledir/spiders/google_directory.py

@@ -0,0 +1,40 @@
+from scrapy.selector import HtmlXPathSelector
+from scrapy.contrib.loader import XPathItemLoader
+from scrapy.contrib_exp.crawlspider import CrawlSpider, Rule
+
+from googledir.items import GoogledirItem
+
+class GoogleDirectorySpider(CrawlSpider):
+
+    domain_name = 'directory.google.com'
+    start_urls = ['http://directory.google.com/']
+
+    rules = (
+        # search for categories pattern and follow links
+        Rule(r'/[A-Z][a-zA-Z_/]+$', 'parse_category', follow=True),
+        )
+
+    def parse_category(self, response):
+        # The main selector we're using to extract data from the page
+        main_selector = HtmlXPathSelector(response)
+
+        # The XPath to website links in the directory page
+        xpath = '//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font'
+
+        # Get a list of (sub) selectors to each website node pointed by the XPath
+        sub_selectors = main_selector.select(xpath)
+
+        # Iterate over the sub-selectors to extract data for each website
+        for selector in sub_selectors:
+            item = GoogledirItem()
+
+            l = XPathItemLoader(item=item, selector=selector)
+            l.add_xpath('name', 'a/text()')
+            l.add_xpath('url', 'a/@href')
+            l.add_xpath('description', 'font[2]/text()')
+
+            # Here we populate the item and yield it
+            yield l.load_item()
+
+SPIDER = GoogleDirectorySpider()
+

examples/experimental/googledir/scrapy-ctl.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+import os
+os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'googledir.settings')
+
+from scrapy.command.cmdline import execute
+execute()