some cleanup to googledir example project

examples/googledir/googledir/items.py

@@ -3,4 +3,7 @@
 from scrapy.item import ScrapedItem
 
 class GoogledirItem(ScrapedItem):
-    pass
+
+    def __str__(self):
+        return "Google Category: name=%s url=%s" % (self.name, self.url)
+

examples/googledir/googledir/pipelines.py

@@ -1,5 +1,15 @@
-# Define yours item pipelines here
+from scrapy.core.exceptions import DropItem
+
+class FilterWordsPipeline(object):
+    """A pipeline for filtering out items which contain certain words in their
+    description"""
+
+    # put all words in lowercase
+    words_to_filter = ['politics', 'religion']
 
-class GoogledirPipeline(object):
     def process_item(self, domain, item):
-        return item
+        for word in self.words_to_filter:
+            if word in unicode(item.description).lower():
+                raise DropItem("Contains forbidden word: %s" % word)
+        else:
+            return item

examples/googledir/googledir/settings.py

@@ -1,4 +1,4 @@
-# - Scrapy settings for googledir                                    -
+# - Scrapy settings for googledir project
 
 import googledir
 
@@ -13,10 +13,5 @@ TEMPLATES_DIR = '%s/templates' % googledir.__path__[0]
 DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem'
 USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
 
-# uncomment if you want to add your own custom scrapy commands
-#COMMANDS_MODULE = 'googledir.commands'
-#COMMANDS_SETTINGS_MODULE = 'googledir.conf.commands'
+ITEM_PIPELINES = ['googledir.pipelines.FilterWordsPipeline']
 
-# global mail sending settings
-#MAIL_HOST = 'localhost'
-#MAIL_FROM = 'scrapybot@localhost'

examples/googledir/googledir/spiders/google_directory.py

@@ -1,17 +1,15 @@
-# -*- coding: utf8 -*-
-import re
-
 from scrapy.xpath import HtmlXPathSelector
-from scrapy.link.extractors import RegexLinkExtractor
+from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
 from scrapy.contrib.spiders import CrawlSpider, Rule
 from googledir.items import GoogledirItem
 
 class GoogleDirectorySpider(CrawlSpider):
-    domain_name = 'google.com'
-    start_urls = ['http://www.google.com/dirhp']
+
+    domain_name = 'directory.google.com'
+    start_urls = ['http://directory.google.com/']
 
     rules = (
-        Rule(RegexLinkExtractor(allow='google.com/[A-Z][a-zA-Z_/]+$'),
+        Rule(SgmlLinkExtractor(allow='directory.google.com/[A-Z][a-zA-Z_/]+$'),
             'parse_category',
             follow=True,
         ),
@@ -29,7 +27,7 @@ class GoogleDirectorySpider(CrawlSpider):
 
             item.name = link.x('a/text()').extract()
             item.url = link.x('a/@href').extract()
-            item.description = link.x('font[2]/text()')
+            item.description = link.x('font[2]/text()').extract()
             yield item
 
 SPIDER = GoogleDirectorySpider()