diff --git a/examples/experimental/googledir/googledir/__init__.py b/examples/experimental/googledir/googledir/__init__.py new file mode 100644 index 000000000..3104ef709 --- /dev/null +++ b/examples/experimental/googledir/googledir/__init__.py @@ -0,0 +1 @@ +# googledir project diff --git a/examples/experimental/googledir/googledir/items.py b/examples/experimental/googledir/googledir/items.py new file mode 100644 index 000000000..decc2c9ba --- /dev/null +++ b/examples/experimental/googledir/googledir/items.py @@ -0,0 +1,16 @@ +# Define here the models for your scraped items +# +# See documentation in: +# http://doc.scrapy.org/topics/items.html + +from scrapy.item import Item, Field + +class GoogledirItem(Item): + + name = Field(default='') + url = Field(default='') + description = Field(default='') + + def __str__(self): + return "Google Category: name=%s url=%s" \ + % (self['name'], self['url']) diff --git a/examples/experimental/googledir/googledir/pipelines.py b/examples/experimental/googledir/googledir/pipelines.py new file mode 100644 index 000000000..f775b254c --- /dev/null +++ b/examples/experimental/googledir/googledir/pipelines.py @@ -0,0 +1,22 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: http://doc.scrapy.org/topics/item-pipeline.html + +from scrapy.core.exceptions import DropItem + +class FilterWordsPipeline(object): + """ + A pipeline for filtering out items which contain certain + words in their description + """ + + # put all words in lowercase + words_to_filter = ['politics', 'religion'] + + def process_item(self, spider, item): + for word in self.words_to_filter: + if word in unicode(item['description']).lower(): + raise DropItem("Contains forbidden word: %s" % word) + else: + return item diff --git a/examples/experimental/googledir/googledir/settings.py b/examples/experimental/googledir/googledir/settings.py new file mode 100644 index 000000000..4e3c11163 --- /dev/null +++ b/examples/experimental/googledir/googledir/settings.py @@ -0,0 +1,21 @@ +# Scrapy settings for googledir project +# +# For simplicity, this file contains only the most important settings by +# default. All the other settings are documented here: +# +# http://doc.scrapy.org/topics/settings.html +# +# Or you can copy and paste them from where they're defined in Scrapy: +# +# scrapy/conf/default_settings.py +# + +BOT_NAME = 'googledir' +BOT_VERSION = '1.0' + +SPIDER_MODULES = ['googledir.spiders'] +NEWSPIDER_MODULE = 'googledir.spiders' +DEFAULT_ITEM_CLASS = 'googledir.items.GoogledirItem' +USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) + +ITEM_PIPELINES = ['googledir.pipelines.FilterWordsPipeline'] diff --git a/examples/experimental/googledir/googledir/spiders/__init__.py b/examples/experimental/googledir/googledir/spiders/__init__.py new file mode 100644 index 000000000..5065ccba5 --- /dev/null +++ b/examples/experimental/googledir/googledir/spiders/__init__.py @@ -0,0 +1,8 @@ +# This package will contain the spiders of your Scrapy project +# +# To create the first spider for your project use this command: +# +# scrapy-ctl.py genspider myspider myspider-domain.com +# +# For more info see: +# http://doc.scrapy.org/topics/spiders.html diff --git a/examples/experimental/googledir/googledir/spiders/google_directory.py b/examples/experimental/googledir/googledir/spiders/google_directory.py new file mode 100644 index 000000000..fdbf3f24a --- /dev/null +++ b/examples/experimental/googledir/googledir/spiders/google_directory.py @@ -0,0 +1,40 @@ +from scrapy.selector import HtmlXPathSelector +from scrapy.contrib.loader import XPathItemLoader +from scrapy.contrib_exp.crawlspider import CrawlSpider, Rule + +from googledir.items import GoogledirItem + +class GoogleDirectorySpider(CrawlSpider): + + domain_name = 'directory.google.com' + start_urls = ['http://directory.google.com/'] + + rules = ( + # search for categories pattern and follow links + Rule(r'/[A-Z][a-zA-Z_/]+$', 'parse_category', follow=True), + ) + + def parse_category(self, response): + # The main selector we're using to extract data from the page + main_selector = HtmlXPathSelector(response) + + # The XPath to website links in the directory page + xpath = '//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font' + + # Get a list of (sub) selectors to each website node pointed by the XPath + sub_selectors = main_selector.select(xpath) + + # Iterate over the sub-selectors to extract data for each website + for selector in sub_selectors: + item = GoogledirItem() + + l = XPathItemLoader(item=item, selector=selector) + l.add_xpath('name', 'a/text()') + l.add_xpath('url', 'a/@href') + l.add_xpath('description', 'font[2]/text()') + + # Here we populate the item and yield it + yield l.load_item() + +SPIDER = GoogleDirectorySpider() + diff --git a/examples/experimental/googledir/scrapy-ctl.py b/examples/experimental/googledir/scrapy-ctl.py new file mode 100644 index 000000000..552421ac3 --- /dev/null +++ b/examples/experimental/googledir/scrapy-ctl.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import os +os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'googledir.settings') + +from scrapy.command.cmdline import execute +execute()