diff --git a/examples/experimental/googledir/googledir/__init__.py b/examples/experimental/googledir/googledir/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/experimental/googledir/googledir/items.py b/examples/experimental/googledir/googledir/items.py deleted file mode 100644 index 0deb766a4..000000000 --- a/examples/experimental/googledir/googledir/items.py +++ /dev/null @@ -1,20 +0,0 @@ -# Define here the models for your scraped items - -from scrapy.contrib_exp import newitem - -from scrapy.contrib_exp.newitem.adaptors import ItemAdaptor, adaptor -from scrapy.contrib_exp.adaptors import extract, strip - - -class GoogledirItem(newitem.Item): - name = newitem.StringField() - url = newitem.StringField() - description = newitem.StringField() - - -class GoogledirItemAdaptor(ItemAdaptor): - item_class = GoogledirItem - - name = adaptor(extract, strip) - url = adaptor(extract, strip) - description = adaptor(extract, strip) diff --git a/examples/experimental/googledir/googledir/pipelines.py b/examples/experimental/googledir/googledir/pipelines.py deleted file mode 100644 index 8e026dbba..000000000 --- a/examples/experimental/googledir/googledir/pipelines.py +++ /dev/null @@ -1,5 +0,0 @@ -# Define yours item pipelines here - -class GoogledirPipeline(object): - def process_item(self, domain, item): - return item diff --git a/examples/experimental/googledir/googledir/settings.py b/examples/experimental/googledir/googledir/settings.py deleted file mode 100644 index 57fd471dd..000000000 --- a/examples/experimental/googledir/googledir/settings.py +++ /dev/null @@ -1,77 +0,0 @@ -import googledir - -# --------------------------------------------------------------------------- -# - Scrapy settings for googledir - -# --------------------------------------------------------------------------- -PROJECT_NAME = 'googledir' - -BOT_NAME = PROJECT_NAME -BOT_VERSION = '1.0' - -SPIDER_MODULES = ['googledir.spiders'] -NEWSPIDER_MODULE = 'googledir.spiders' -TEMPLATES_DIR = '%s/templates' % googledir.__path__[0] -DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem' -USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) - -# The amount of time (in secs) that the downloader should wait before -# downloading consecutive pages from the same spider. This can be used -# to throttle the crawling speed to avoid hitting servers too -# hard. Decimal numbers are supported. Example: -# DOWNLOAD_DELAY = 2.5 -DOWNLOAD_TIMEOUT = 600 - -# use this spider class as default when no spider was found for a given url -#DEFAULT_SPIDER = 'scrapy.contrib.spiders.generic.GenericSpider' - -# uncomment if you want to add your own custom scrapy commands -#COMMANDS_MODULE = 'googledir.commands' -#COMMANDS_SETTINGS_MODULE = 'googledir.conf.commands' - -#Global timeout between sucessive downloads (can be overrided by spider -#attribute download_timeout -#DOWNLOAD_TIMEOUT = 0 - -MYSQL_CONNECTION_SETTINGS = {"charset": "utf8" } -MYSQL_CONNECTION_PING_PERIOD = 600 - -SCHEDULER = 'scrapy.core.scheduler.Scheduler' -SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO - -#HTTPCACHE_DIR = '/tmp/cache2' # if set, enables HTTP cache -#HTTPCACHE_IGNORE_MISSING = 0 # ignore requests not in cache -#HTTPCACHE_SECTORIZE = 1 # sectorize domains to distribute storage among servers - -#STATS_ENABLED = 1 # enable stats -#STATS_CLEANUP = 0 # cleanup domain stats when a domain is closed (saves memory) -#STATS_DEBUG = 0 # log stats on domain closed - -EXTENSIONS = ( - 'scrapy.management.web.WebConsole', - 'scrapy.management.telnet.TelnetConsole', -) - -ITEM_PIPELINES = ( - 'googledir.pipelines.GoogledirPipeline', -) - -#DEPTH_LIMIT = 10 # limit the maximum link depth to follow -#DEPTH_STATS = 1 # enable depth stats - -# Limit URL length. See: http://www.boutell.com/newfaq/misc/urllength.html -URLLENGTH_LIMIT = 2083 - -#WEBCONSOLE_ENABLED = 1 -#WEBCONSOLE_PORT = 8060 # if not set uses a dynamic port - -#TELNETCONSOLE_ENABLED = 1 -#TELNETCONSOLE_PORT = 2020 # if not set uses a dynamic port - -# global mail sending settings -#MAIL_HOST = 'localhost' -#MAIL_FROM = 'scrapybot@localhost' - -# scrapy webservice -WS_ENABLED = 0 - -SPIDERPROFILER_ENABLED = 0 diff --git a/examples/experimental/googledir/googledir/spiders/__init__.py b/examples/experimental/googledir/googledir/spiders/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/experimental/googledir/googledir/spiders/google_directory.py b/examples/experimental/googledir/googledir/spiders/google_directory.py deleted file mode 100644 index 7d7948f4e..000000000 --- a/examples/experimental/googledir/googledir/spiders/google_directory.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf8 -*- -import re - -from scrapy.xpath import HtmlXPathSelector -from scrapy.contrib.spiders import CrawlSpider, rule -from scrapy.utils.misc import items_to_csv - -from googledir.items import GoogledirItem, GoogledirItemAdaptor - -class GoogleDirectorySpider(CrawlSpider): - domain_name = 'google.com' - start_urls = ['http://www.google.com/dirhp'] - - rules = ( - rule('google.com/[A-Z][a-zA-Z_/]+$', 'parse_category', follow=True), - ) - csv_file = open('scraped_items.csv', 'ab+') - - def parse_category(self, response): - # The selector we're going to use in order to extract data from the page - hxs = HtmlXPathSelector(response) - - # The path to website links in directory page - links = hxs.x('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') - - for link in links: - extractor = GoogledirItemAdaptor() - extractor.name = link.x('a/text()') - extractor.url = link.x('a/@href') - extractor.description = link.x('font[2]/text()') - item = extractor.item_instance - yield item - -SPIDER = GoogleDirectorySpider() diff --git a/examples/experimental/googledir/googledir/templates/spider_basic.tmpl b/examples/experimental/googledir/googledir/templates/spider_basic.tmpl deleted file mode 100644 index 75f2478d5..000000000 --- a/examples/experimental/googledir/googledir/templates/spider_basic.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -from scrapy.spider import BaseSpider - -class $classname(BaseSpider): - domain_name = "$site" - start_urls = ( - 'http://www.$site/', - ) - - def parse(self, response): - return () - -SPIDER = $classname() - diff --git a/examples/experimental/googledir/googledir/templates/spider_crawl.tmpl b/examples/experimental/googledir/googledir/templates/spider_crawl.tmpl deleted file mode 100644 index 9f777b231..000000000 --- a/examples/experimental/googledir/googledir/templates/spider_crawl.tmpl +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf8 -*- -import re - -from scrapy.xpath import HtmlXPathSelector -from scrapy.link.extractors import RegexLinkExtractor -from scrapy.contrib.spiders import CrawlSpider, Rule -from $project_name.items import ${ProjectName}Item - -class $classname(CrawlSpider): - domain_name = '$site' - start_urls = ['http://www.$site/'] - - rules = ( - Rule(RegexLinkExtractor(allow=(r'Items/', )), 'parse_item', follow=True), - ) - - def parse_item(self, response): - i = ${ProjectName}Item() - #xs = HtmlXPathSelector(response) - #i.attribute('site_id', xs.x('//input[@id="sid"]/@value')) - #i.attribute('name', xs.x('//div[@id="name"]')) - #i.attribute('description', xs.x('//div[@id="description"]')) - return [i] - -SPIDER = $classname() diff --git a/examples/experimental/googledir/googledir/templates/spider_csvfeed.tmpl b/examples/experimental/googledir/googledir/templates/spider_csvfeed.tmpl deleted file mode 100644 index 2d6ec5991..000000000 --- a/examples/experimental/googledir/googledir/templates/spider_csvfeed.tmpl +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf8 -*- -from scrapy.contrib.spiders import CSVFeedSpider -from $project_name.items import ${ProjectName}Item - -class $classname(CSVFeedSpider): - domain_name = '$site' - start_urls = ['http://www.$site/feed.csv'] - # headers = ['id', 'name', 'description', 'image_link'] - # delimiter = '\t' - - # Do any adaptations you need here - #def adapt_response(self, response): - # return response - - def parse_row(self, response, row): - i = ${ProjectName}Item() - #i.attribute('url', row['url']) - #i.attribute('name', row['name']) - #i.attribute('description', row['description']) - return i - -SPIDER = $classname() diff --git a/examples/experimental/googledir/googledir/templates/spider_xmlfeed.tmpl b/examples/experimental/googledir/googledir/templates/spider_xmlfeed.tmpl deleted file mode 100644 index 395204cf5..000000000 --- a/examples/experimental/googledir/googledir/templates/spider_xmlfeed.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf8 -*- -from scrapy.contrib.spiders import XMLFeedSpider -from $project_name.items import ${ProjectName}Item - -class $classname(XMLFeedSpider): - domain_name = '$site' - start_urls = ['http://www.$site/feed.xml'] - - def parse_item(self, response, xSel): - i = ${ProjectName}Item() - #i.attribute('url', xSel('url')) - #i.attribute('name', xSel('name')) - #i.attribute('description', xSel('description')) - return i - -SPIDER = $classname() diff --git a/examples/experimental/googledir/scrapy-ctl.py b/examples/experimental/googledir/scrapy-ctl.py deleted file mode 100755 index 8820b5d37..000000000 --- a/examples/experimental/googledir/scrapy-ctl.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python - -import os -os.environ.setdefault('SCRAPYSETTINGS_MODULE', 'googledir.settings') - -from scrapy.command.cmdline import execute -execute()