add new googledir example project with new structure

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40854

scrapy/trunk/examples/googledir/googledir/items.py

@@ -0,0 +1,13 @@
+# Define here the models for your scraped items
+
+from scrapy.contrib.item import RobustScrapedItem
+
+class GoogledirItem(RobustScrapedItem):
+    """Directory website link"""
+
+    ATTRIBUTES = {
+        'guid': basestring,
+        'name': basestring,
+        'url': basestring,
+        'description': basestring,
+        }

scrapy/trunk/examples/googledir/googledir/pipelines.py

@@ -0,0 +1,5 @@
+# Define yours item pipelines here
+
+class GoogledirPipeline(object):
+    def process_item(self, domain, item):
+        return item

scrapy/trunk/examples/googledir/googledir/settings.py

@@ -0,0 +1,106 @@
+import googledir
+
+# ---------------------------------------------------------------------------
+# - Scrapy settings for googledir                                    -
+# ---------------------------------------------------------------------------
+PROJECT_NAME = 'googledir'
+
+BOT_NAME = PROJECT_NAME
+BOT_VERSION = '1.0'
+
+SPIDER_MODULES = ['googledir.spiders']
+NEWSPIDER_MODULE = 'googledir.spiders'
+TEMPLATES_DIR = '%s/templates' % googledir.__path__[0]
+ENABLED_SPIDERS_FILE = '%s/conf/enabled_spiders.list' % googledir.__path__[0]
+DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem'
+USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
+
+# The amount of time (in secs) that the downloader should wait before
+# downloading consecutive pages from the same spider. This can be used
+# to throttle the crawling speed to avoid hitting servers too
+# hard. Decimal numbers are supported. Example:
+# DOWNLOAD_DELAY = 2.5
+DOWNLOAD_TIMEOUT = 600
+
+# use this spider class as default when no spider was found for a given url
+#DEFAULT_SPIDER = 'scrapy.contrib.spiders.generic.GenericSpider'
+
+# uncomment if you want to add your own custom scrapy commands
+#COMMANDS_MODULE = 'googledir.commands'
+#COMMANDS_SETTINGS_MODULE = 'googledir.conf.commands'
+
+#Global timeout between sucessive downloads (can be overrided by spider
+#attribute download_timeout
+#DOWNLOAD_TIMEOUT = 0
+
+MYSQL_CONNECTION_SETTINGS = {"charset": "utf8" }
+MYSQL_CONNECTION_PING_PERIOD = 600
+
+SCHEDULER = 'scrapy.core.scheduler.Scheduler'
+SCHEDULER_ORDER = 'BFO'   # available orders: BFO (default), DFO
+
+#CACHE2_DIR = '/tmp/cache2'  # if set, enables HTTP cache
+#CACHE2_IGNORE_MISSING = 0   # ignore requests not in cache
+#CACHE2_SECTORIZE = 1         # sectorize domains to distribute storage among servers
+
+#STATS_ENABLED = 1   # enable stats
+#STATS_CLEANUP = 0   # cleanup domain stats when a domain is closed (saves memory)
+#STATS_DEBUG = 0     # log stats on domain closed
+
+EXTENSIONS = (
+    'scrapy.management.web.WebConsole',
+    'scrapy.management.telnet.TelnetConsole',
+)
+
+DOWNLOADER_MIDDLEWARES = (
+    # Engine side
+    'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware',
+    'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware',
+    'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware',
+    'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware',
+    'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware',
+    'scrapy.contrib.downloadermiddleware.common.CommonMiddleware',
+    'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware',
+    'scrapy.contrib.downloadermiddleware.compression.CompressionMiddleware',
+    'scrapy.contrib.downloadermiddleware.debug.CrawlDebug',
+    'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware',
+    # Downloader side
+)
+
+SPIDER_MIDDLEWARES = (
+    # Engine side
+    'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware',
+    'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware',
+    'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware',
+    'scrapy.contrib.spidermiddleware.referer.RefererMiddleware',
+    'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware',
+    'scrapy.contrib.spidermiddleware.depth.DepthMiddleware',
+    'scrapy.contrib.spidermiddleware.urlfilter.UrlFilterMiddleware',
+    'scrapy.contrib.spidermiddleware.duplicatesfilter.DuplicatesFilterMiddleware',
+    # Spider side
+)
+
+ITEM_PIPELINES = (
+    'googledir.pipelines.GoogledirPipeline',
+)
+
+#DEPTH_LIMIT = 10  # limit the maximum link depth to follow
+#DEPTH_STATS = 1    # enable depth stats
+
+# Limit URL length. See: http://www.boutell.com/newfaq/misc/urllength.html
+URLLENGTH_LIMIT = 2083
+
+#WEBCONSOLE_ENABLED = 1
+#WEBCONSOLE_PORT = 8060  # if not set uses a dynamic port
+
+#TELNETCONSOLE_ENABLED = 1
+#TELNETCONSOLE_PORT = 2020  # if not set uses a dynamic port
+
+# global mail sending settings
+#MAIL_HOST = 'localhost'
+#MAIL_FROM = 'scrapybot@localhost'
+
+# scrapy webservice
+WS_ENABLED = 0
+
+SPIDERPROFILER_ENABLED = 0

scrapy/trunk/examples/googledir/googledir/spiders/__init__.py

@@ -0,0 +1 @@
+# Place here all your scrapy spiders

scrapy/trunk/examples/googledir/googledir/spiders/dropin.cache

@@ -0,0 +1,45 @@
+(dp1
+S'google_directory'
+p2
+ccopy_reg
+_reconstructor
+p3
+(ctwisted.plugin
+CachedDropin
+p4
+c__builtin__
+object
+p5
+NtRp6
+(dp7
+S'moduleName'
+p8
+S'googledir.spiders.google_directory'
+p9
+sS'description'
+p10
+NsS'plugins'
+p11
+(lp12
+g3
+(ctwisted.plugin
+CachedPlugin
+p13
+g5
+NtRp14
+(dp15
+S'provided'
+p16
+(lp17
+cscrapy.spider.models
+ISpider
+p18
+asS'dropin'
+p19
+g6
+sS'name'
+p20
+S'SPIDER'
+p21
+sg10
+Nsbasbs.

scrapy/trunk/examples/googledir/googledir/spiders/google_directory.py

@@ -0,0 +1,48 @@
+# -*- coding: utf8 -*-
+import re
+
+from scrapy.xpath import HtmlXPathSelector
+from scrapy.link.extractors import RegexLinkExtractor
+from scrapy.contrib.spiders import CrawlSpider, Rule
+from scrapy.contrib_exp import adaptors
+from scrapy.utils.misc import items_to_csv
+from googledir.items import GoogledirItem
+
+class GoogleDirectorySpider(CrawlSpider):
+    domain_name = 'google.com'
+    start_urls = ['http://www.google.com/dirhp']
+
+    rules = (
+        Rule(RegexLinkExtractor(allow=('google.com/[A-Z][a-zA-Z_/]+$',),),
+            'parse_category',
+            follow=True,
+        ),
+    )
+    csv_file = open('scraped_items.csv', 'ab+')
+    
+    def parse_category(self, response):
+        # The selector we're going to use in order to extract data from the page
+        hxs = HtmlXPathSelector(response)
+
+        # The path to website links in directory page
+        links = hxs.x('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font')
+
+        # The list of functions to apply to an attribute before assigning its value
+        adaptor_pipe = [adaptors.extract, adaptors.delist(''), adaptors.strip]
+        adaptor_map = {
+            'name': adaptor_pipe,
+            'url': adaptor_pipe,
+            'description': adaptor_pipe,
+            }
+
+        for link in links:
+            item = GoogledirItem()
+            item.set_adaptors(adaptor_map)
+
+            item.attribute('name', link.x('a/text()'))
+            item.attribute('url', link.x('a/@href'))
+            item.attribute('description', link.x('font[2]/text()'))
+            items_to_csv(self.csv_file, [item])
+            yield item
+
+SPIDER = GoogleDirectorySpider()

scrapy/trunk/examples/googledir/googledir/templates/spider_basic.tmpl

@@ -0,0 +1,13 @@
+from scrapy.spider import BaseSpider
+
+class $classname(BaseSpider):
+    domain_name = "$site"
+    start_urls = (
+        'http://www.$site/',
+        )
+
+    def parse(self, response):
+        return ()
+
+SPIDER = $classname()
+

scrapy/trunk/examples/googledir/googledir/templates/spider_crawl.tmpl

@@ -0,0 +1,25 @@
+# -*- coding: utf8 -*-
+import re
+
+from scrapy.xpath import HtmlXPathSelector
+from scrapy.link.extractors import RegexLinkExtractor
+from scrapy.contrib.spiders import CrawlSpider, Rule
+from $project_name.items import ${ProjectName}Item
+
+class $classname(CrawlSpider):
+    domain_name = '$site'
+    start_urls = ['http://www.$site/']
+
+    rules = (
+        Rule(RegexLinkExtractor(allow=(r'Items/', )), 'parse_item', follow=True),
+    )
+
+    def parse_item(self, response):
+        i = ${ProjectName}Item()
+        #xs = HtmlXPathSelector(response)
+        #i.attribute('site_id', xs.x('//input[@id="sid"]/@value'))
+        #i.attribute('name', xs.x('//div[@id="name"]'))
+        #i.attribute('description', xs.x('//div[@id="description"]'))
+        return [i]
+
+SPIDER = $classname()

scrapy/trunk/examples/googledir/googledir/templates/spider_csvfeed.tmpl

@@ -0,0 +1,22 @@
+# -*- coding: utf8 -*-
+from scrapy.contrib.spiders import CSVFeedSpider
+from $project_name.items import ${ProjectName}Item
+
+class $classname(CSVFeedSpider):
+    domain_name = '$site'
+    start_urls = ['http://www.$site/feed.csv']
+    # headers = ['id', 'name', 'description', 'image_link']
+    # delimiter = '\t'
+
+    # Do any adaptations you need here
+    #def adapt_response(self, response):
+    #    return response
+
+    def parse_row(self, response, row):
+        i = ${ProjectName}Item()
+        #i.attribute('url', row['url'])
+        #i.attribute('name', row['name'])
+        #i.attribute('description', row['description'])
+        return i
+
+SPIDER = $classname()

scrapy/trunk/examples/googledir/googledir/templates/spider_xmlfeed.tmpl

@@ -0,0 +1,16 @@
+# -*- coding: utf8 -*-
+from scrapy.contrib.spiders import XMLFeedSpider
+from $project_name.items import ${ProjectName}Item
+
+class $classname(XMLFeedSpider):
+    domain_name = '$site'
+    start_urls = ['http://www.$site/feed.xml']
+
+    def parse_item(self, response, xSel):
+        i = ${ProjectName}Item()
+        #i.attribute('url', xSel('url'))
+        #i.attribute('name', xSel('name'))
+        #i.attribute('description', xSel('description'))
+        return i
+
+SPIDER = $classname()

scrapy/trunk/examples/googledir/scrapy-ctl.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+import os
+os.environ.setdefault('SCRAPYSETTINGS_MODULE', 'googledir.settings')
+
+from scrapy.command.cmdline import execute
+execute()