From 059bf613fab0f3e138caeb1273e132194504cb8f Mon Sep 17 00:00:00 2001 From: Ismael Carnales Date: Fri, 20 Feb 2009 12:02:56 +0000 Subject: [PATCH] added example project that uses newitem and ItemExtractor --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40895 --- .../googledir/googledir/__init__.py | 0 .../contrib_exp/googledir/googledir/items.py | 20 ++++ .../googledir/googledir/pipelines.py | 5 + .../googledir/googledir/settings.py | 106 ++++++++++++++++++ .../googledir/googledir/spiders/__init__.py | 0 .../googledir/spiders/google_directory.py | 34 ++++++ .../googledir/templates/spider_basic.tmpl | 13 +++ .../googledir/templates/spider_crawl.tmpl | 25 +++++ .../googledir/templates/spider_csvfeed.tmpl | 22 ++++ .../googledir/templates/spider_xmlfeed.tmpl | 16 +++ .../contrib_exp/googledir/scrapy-ctl.py | 7 ++ 11 files changed, 248 insertions(+) create mode 100644 scrapy/trunk/examples/contrib_exp/googledir/googledir/__init__.py create mode 100644 scrapy/trunk/examples/contrib_exp/googledir/googledir/items.py create mode 100644 scrapy/trunk/examples/contrib_exp/googledir/googledir/pipelines.py create mode 100644 scrapy/trunk/examples/contrib_exp/googledir/googledir/settings.py create mode 100644 scrapy/trunk/examples/contrib_exp/googledir/googledir/spiders/__init__.py create mode 100644 scrapy/trunk/examples/contrib_exp/googledir/googledir/spiders/google_directory.py create mode 100644 scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_basic.tmpl create mode 100644 scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_crawl.tmpl create mode 100644 scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_csvfeed.tmpl create mode 100644 scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_xmlfeed.tmpl create mode 100755 scrapy/trunk/examples/contrib_exp/googledir/scrapy-ctl.py diff --git a/scrapy/trunk/examples/contrib_exp/googledir/googledir/__init__.py b/scrapy/trunk/examples/contrib_exp/googledir/googledir/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/examples/contrib_exp/googledir/googledir/items.py b/scrapy/trunk/examples/contrib_exp/googledir/googledir/items.py new file mode 100644 index 000000000..857e05afc --- /dev/null +++ b/scrapy/trunk/examples/contrib_exp/googledir/googledir/items.py @@ -0,0 +1,20 @@ +# Define here the models for your scraped items + +from scrapy.contrib_exp import newitem + +from scrapy.contrib_exp.newitem import extractors +from scrapy.contrib_exp import adaptors + + +class GoogledirItem(newitem.Item): + name = newitem.StringField() + url = newitem.StringField() + description = newitem.StringField() + + +class GoogledirItemExtractor(extractors.ItemExtractor): + item_class = GoogledirItem + + name = extractors.ExtractorField([adaptors.extract, adaptors.strip]) + url = extractors.ExtractorField([adaptors.extract, adaptors.strip]) + description = extractors.ExtractorField([adaptors.extract, adaptors.strip]) diff --git a/scrapy/trunk/examples/contrib_exp/googledir/googledir/pipelines.py b/scrapy/trunk/examples/contrib_exp/googledir/googledir/pipelines.py new file mode 100644 index 000000000..8e026dbba --- /dev/null +++ b/scrapy/trunk/examples/contrib_exp/googledir/googledir/pipelines.py @@ -0,0 +1,5 @@ +# Define yours item pipelines here + +class GoogledirPipeline(object): + def process_item(self, domain, item): + return item diff --git a/scrapy/trunk/examples/contrib_exp/googledir/googledir/settings.py b/scrapy/trunk/examples/contrib_exp/googledir/googledir/settings.py new file mode 100644 index 000000000..32153f077 --- /dev/null +++ b/scrapy/trunk/examples/contrib_exp/googledir/googledir/settings.py @@ -0,0 +1,106 @@ +import googledir + +# --------------------------------------------------------------------------- +# - Scrapy settings for googledir - +# --------------------------------------------------------------------------- +PROJECT_NAME = 'googledir' + +BOT_NAME = PROJECT_NAME +BOT_VERSION = '1.0' + +SPIDER_MODULES = ['googledir.spiders'] +NEWSPIDER_MODULE = 'googledir.spiders' +TEMPLATES_DIR = '%s/templates' % googledir.__path__[0] +ENABLED_SPIDERS_FILE = '%s/conf/enabled_spiders.list' % googledir.__path__[0] +DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem' +USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) + +# The amount of time (in secs) that the downloader should wait before +# downloading consecutive pages from the same spider. This can be used +# to throttle the crawling speed to avoid hitting servers too +# hard. Decimal numbers are supported. Example: +# DOWNLOAD_DELAY = 2.5 +DOWNLOAD_TIMEOUT = 600 + +# use this spider class as default when no spider was found for a given url +#DEFAULT_SPIDER = 'scrapy.contrib.spiders.generic.GenericSpider' + +# uncomment if you want to add your own custom scrapy commands +#COMMANDS_MODULE = 'googledir.commands' +#COMMANDS_SETTINGS_MODULE = 'googledir.conf.commands' + +#Global timeout between sucessive downloads (can be overrided by spider +#attribute download_timeout +#DOWNLOAD_TIMEOUT = 0 + +MYSQL_CONNECTION_SETTINGS = {"charset": "utf8" } +MYSQL_CONNECTION_PING_PERIOD = 600 + +SCHEDULER = 'scrapy.core.scheduler.Scheduler' +SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO + +#CACHE2_DIR = '/tmp/cache2' # if set, enables HTTP cache +#CACHE2_IGNORE_MISSING = 0 # ignore requests not in cache +#CACHE2_SECTORIZE = 1 # sectorize domains to distribute storage among servers + +#STATS_ENABLED = 1 # enable stats +#STATS_CLEANUP = 0 # cleanup domain stats when a domain is closed (saves memory) +#STATS_DEBUG = 0 # log stats on domain closed + +EXTENSIONS = ( + 'scrapy.management.web.WebConsole', + 'scrapy.management.telnet.TelnetConsole', +) + +DOWNLOADER_MIDDLEWARES = ( + # Engine side + 'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware', + 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware', + 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware', + 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware', + 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware', + 'scrapy.contrib.downloadermiddleware.common.CommonMiddleware', + 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware', + 'scrapy.contrib.downloadermiddleware.compression.CompressionMiddleware', + 'scrapy.contrib.downloadermiddleware.debug.CrawlDebug', + 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware', + # Downloader side +) + +SPIDER_MIDDLEWARES = ( + # Engine side + 'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware', + 'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware', + 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware', + 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware', + 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware', + 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware', + 'scrapy.contrib.spidermiddleware.urlfilter.UrlFilterMiddleware', + 'scrapy.contrib.spidermiddleware.duplicatesfilter.DuplicatesFilterMiddleware', + # Spider side +) + +ITEM_PIPELINES = ( + 'googledir.pipelines.GoogledirPipeline', +) + +#DEPTH_LIMIT = 10 # limit the maximum link depth to follow +#DEPTH_STATS = 1 # enable depth stats + +# Limit URL length. See: http://www.boutell.com/newfaq/misc/urllength.html +URLLENGTH_LIMIT = 2083 + +#WEBCONSOLE_ENABLED = 1 +#WEBCONSOLE_PORT = 8060 # if not set uses a dynamic port + +#TELNETCONSOLE_ENABLED = 1 +#TELNETCONSOLE_PORT = 2020 # if not set uses a dynamic port + +# global mail sending settings +#MAIL_HOST = 'localhost' +#MAIL_FROM = 'scrapybot@localhost' + +# scrapy webservice +WS_ENABLED = 0 + +SPIDERPROFILER_ENABLED = 0 diff --git a/scrapy/trunk/examples/contrib_exp/googledir/googledir/spiders/__init__.py b/scrapy/trunk/examples/contrib_exp/googledir/googledir/spiders/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/examples/contrib_exp/googledir/googledir/spiders/google_directory.py b/scrapy/trunk/examples/contrib_exp/googledir/googledir/spiders/google_directory.py new file mode 100644 index 000000000..1f735f44a --- /dev/null +++ b/scrapy/trunk/examples/contrib_exp/googledir/googledir/spiders/google_directory.py @@ -0,0 +1,34 @@ +# -*- coding: utf8 -*- +import re + +from scrapy.xpath import HtmlXPathSelector +from scrapy.contrib.spiders import CrawlSpider, rule +from scrapy.utils.misc import items_to_csv + +from googledir.items import GoogledirItem, GoogledirItemExtractor + +class GoogleDirectorySpider(CrawlSpider): + domain_name = 'google.com' + start_urls = ['http://www.google.com/dirhp'] + + rules = ( + rule('google.com/[A-Z][a-zA-Z_/]+$', 'parse_category', follow=True), + ) + csv_file = open('scraped_items.csv', 'ab+') + + def parse_category(self, response): + # The selector we're going to use in order to extract data from the page + hxs = HtmlXPathSelector(response) + + # The path to website links in directory page + links = hxs.x('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') + + for link in links: + extractor = GoogledirItemExtractor() + extractor.name = link.x('a/text()') + extractor.url = link.x('a/@href') + extractor.description = link.x('font[2]/text()') + item = extractor.item_instance + yield item + +SPIDER = GoogleDirectorySpider() diff --git a/scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_basic.tmpl b/scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_basic.tmpl new file mode 100644 index 000000000..75f2478d5 --- /dev/null +++ b/scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_basic.tmpl @@ -0,0 +1,13 @@ +from scrapy.spider import BaseSpider + +class $classname(BaseSpider): + domain_name = "$site" + start_urls = ( + 'http://www.$site/', + ) + + def parse(self, response): + return () + +SPIDER = $classname() + diff --git a/scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_crawl.tmpl b/scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_crawl.tmpl new file mode 100644 index 000000000..9f777b231 --- /dev/null +++ b/scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_crawl.tmpl @@ -0,0 +1,25 @@ +# -*- coding: utf8 -*- +import re + +from scrapy.xpath import HtmlXPathSelector +from scrapy.link.extractors import RegexLinkExtractor +from scrapy.contrib.spiders import CrawlSpider, Rule +from $project_name.items import ${ProjectName}Item + +class $classname(CrawlSpider): + domain_name = '$site' + start_urls = ['http://www.$site/'] + + rules = ( + Rule(RegexLinkExtractor(allow=(r'Items/', )), 'parse_item', follow=True), + ) + + def parse_item(self, response): + i = ${ProjectName}Item() + #xs = HtmlXPathSelector(response) + #i.attribute('site_id', xs.x('//input[@id="sid"]/@value')) + #i.attribute('name', xs.x('//div[@id="name"]')) + #i.attribute('description', xs.x('//div[@id="description"]')) + return [i] + +SPIDER = $classname() diff --git a/scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_csvfeed.tmpl b/scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_csvfeed.tmpl new file mode 100644 index 000000000..2d6ec5991 --- /dev/null +++ b/scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_csvfeed.tmpl @@ -0,0 +1,22 @@ +# -*- coding: utf8 -*- +from scrapy.contrib.spiders import CSVFeedSpider +from $project_name.items import ${ProjectName}Item + +class $classname(CSVFeedSpider): + domain_name = '$site' + start_urls = ['http://www.$site/feed.csv'] + # headers = ['id', 'name', 'description', 'image_link'] + # delimiter = '\t' + + # Do any adaptations you need here + #def adapt_response(self, response): + # return response + + def parse_row(self, response, row): + i = ${ProjectName}Item() + #i.attribute('url', row['url']) + #i.attribute('name', row['name']) + #i.attribute('description', row['description']) + return i + +SPIDER = $classname() diff --git a/scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_xmlfeed.tmpl b/scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_xmlfeed.tmpl new file mode 100644 index 000000000..395204cf5 --- /dev/null +++ b/scrapy/trunk/examples/contrib_exp/googledir/googledir/templates/spider_xmlfeed.tmpl @@ -0,0 +1,16 @@ +# -*- coding: utf8 -*- +from scrapy.contrib.spiders import XMLFeedSpider +from $project_name.items import ${ProjectName}Item + +class $classname(XMLFeedSpider): + domain_name = '$site' + start_urls = ['http://www.$site/feed.xml'] + + def parse_item(self, response, xSel): + i = ${ProjectName}Item() + #i.attribute('url', xSel('url')) + #i.attribute('name', xSel('name')) + #i.attribute('description', xSel('description')) + return i + +SPIDER = $classname() diff --git a/scrapy/trunk/examples/contrib_exp/googledir/scrapy-ctl.py b/scrapy/trunk/examples/contrib_exp/googledir/scrapy-ctl.py new file mode 100755 index 000000000..8820b5d37 --- /dev/null +++ b/scrapy/trunk/examples/contrib_exp/googledir/scrapy-ctl.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import os +os.environ.setdefault('SCRAPYSETTINGS_MODULE', 'googledir.settings') + +from scrapy.command.cmdline import execute +execute()