diff --git a/examples/googledir/googledir/items.py b/examples/googledir/googledir/items.py index 60125a102..387c3765f 100644 --- a/examples/googledir/googledir/items.py +++ b/examples/googledir/googledir/items.py @@ -3,4 +3,7 @@ from scrapy.item import ScrapedItem class GoogledirItem(ScrapedItem): - pass + + def __str__(self): + return "Google Category: name=%s url=%s" % (self.name, self.url) + diff --git a/examples/googledir/googledir/pipelines.py b/examples/googledir/googledir/pipelines.py index 8e026dbba..2ac385dab 100644 --- a/examples/googledir/googledir/pipelines.py +++ b/examples/googledir/googledir/pipelines.py @@ -1,5 +1,15 @@ -# Define yours item pipelines here +from scrapy.core.exceptions import DropItem + +class FilterWordsPipeline(object): + """A pipeline for filtering out items which contain certain words in their + description""" + + # put all words in lowercase + words_to_filter = ['politics', 'religion'] -class GoogledirPipeline(object): def process_item(self, domain, item): - return item + for word in self.words_to_filter: + if word in unicode(item.description).lower(): + raise DropItem("Contains forbidden word: %s" % word) + else: + return item diff --git a/examples/googledir/googledir/settings.py b/examples/googledir/googledir/settings.py index 38d06fe31..640530d5a 100644 --- a/examples/googledir/googledir/settings.py +++ b/examples/googledir/googledir/settings.py @@ -1,4 +1,4 @@ -# - Scrapy settings for googledir - +# - Scrapy settings for googledir project import googledir @@ -13,10 +13,5 @@ TEMPLATES_DIR = '%s/templates' % googledir.__path__[0] DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem' USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) -# uncomment if you want to add your own custom scrapy commands -#COMMANDS_MODULE = 'googledir.commands' -#COMMANDS_SETTINGS_MODULE = 'googledir.conf.commands' +ITEM_PIPELINES = ['googledir.pipelines.FilterWordsPipeline'] -# global mail sending settings -#MAIL_HOST = 'localhost' -#MAIL_FROM = 'scrapybot@localhost' diff --git a/examples/googledir/googledir/spiders/google_directory.py b/examples/googledir/googledir/spiders/google_directory.py index ba6f236f9..d28f92d95 100644 --- a/examples/googledir/googledir/spiders/google_directory.py +++ b/examples/googledir/googledir/spiders/google_directory.py @@ -1,17 +1,15 @@ -# -*- coding: utf8 -*- -import re - from scrapy.xpath import HtmlXPathSelector -from scrapy.link.extractors import RegexLinkExtractor +from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor from scrapy.contrib.spiders import CrawlSpider, Rule from googledir.items import GoogledirItem class GoogleDirectorySpider(CrawlSpider): - domain_name = 'google.com' - start_urls = ['http://www.google.com/dirhp'] + + domain_name = 'directory.google.com' + start_urls = ['http://directory.google.com/'] rules = ( - Rule(RegexLinkExtractor(allow='google.com/[A-Z][a-zA-Z_/]+$'), + Rule(SgmlLinkExtractor(allow='directory.google.com/[A-Z][a-zA-Z_/]+$'), 'parse_category', follow=True, ), @@ -29,7 +27,7 @@ class GoogleDirectorySpider(CrawlSpider): item.name = link.x('a/text()').extract() item.url = link.x('a/@href').extract() - item.description = link.x('font[2]/text()') + item.description = link.x('font[2]/text()').extract() yield item SPIDER = GoogleDirectorySpider()