mirror of https://github.com/scrapy/scrapy.git
removed obsolete code
This commit is contained in:
Pablo Hoffman
parent
4a10e1dabd
commit
6fcbd03bff
|
|
@ -1,20 +0,0 @@
|
|||
# Define here the models for your scraped items
|
||||
|
||||
from scrapy.contrib_exp import newitem
|
||||
|
||||
from scrapy.contrib_exp.newitem.adaptors import ItemAdaptor, adaptor
|
||||
from scrapy.contrib_exp.adaptors import extract, strip
|
||||
|
||||
|
||||
class GoogledirItem(newitem.Item):
|
||||
name = newitem.StringField()
|
||||
url = newitem.StringField()
|
||||
description = newitem.StringField()
|
||||
|
||||
|
||||
class GoogledirItemAdaptor(ItemAdaptor):
|
||||
item_class = GoogledirItem
|
||||
|
||||
name = adaptor(extract, strip)
|
||||
url = adaptor(extract, strip)
|
||||
description = adaptor(extract, strip)
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
# Define yours item pipelines here
|
||||
|
||||
class GoogledirPipeline(object):
|
||||
def process_item(self, domain, item):
|
||||
return item
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
import googledir
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# - Scrapy settings for googledir -
|
||||
# ---------------------------------------------------------------------------
|
||||
PROJECT_NAME = 'googledir'
|
||||
|
||||
BOT_NAME = PROJECT_NAME
|
||||
BOT_VERSION = '1.0'
|
||||
|
||||
SPIDER_MODULES = ['googledir.spiders']
|
||||
NEWSPIDER_MODULE = 'googledir.spiders'
|
||||
TEMPLATES_DIR = '%s/templates' % googledir.__path__[0]
|
||||
DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem'
|
||||
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
||||
|
||||
# The amount of time (in secs) that the downloader should wait before
|
||||
# downloading consecutive pages from the same spider. This can be used
|
||||
# to throttle the crawling speed to avoid hitting servers too
|
||||
# hard. Decimal numbers are supported. Example:
|
||||
# DOWNLOAD_DELAY = 2.5
|
||||
DOWNLOAD_TIMEOUT = 600
|
||||
|
||||
# use this spider class as default when no spider was found for a given url
|
||||
#DEFAULT_SPIDER = 'scrapy.contrib.spiders.generic.GenericSpider'
|
||||
|
||||
# uncomment if you want to add your own custom scrapy commands
|
||||
#COMMANDS_MODULE = 'googledir.commands'
|
||||
#COMMANDS_SETTINGS_MODULE = 'googledir.conf.commands'
|
||||
|
||||
#Global timeout between sucessive downloads (can be overrided by spider
|
||||
#attribute download_timeout
|
||||
#DOWNLOAD_TIMEOUT = 0
|
||||
|
||||
MYSQL_CONNECTION_SETTINGS = {"charset": "utf8" }
|
||||
MYSQL_CONNECTION_PING_PERIOD = 600
|
||||
|
||||
SCHEDULER = 'scrapy.core.scheduler.Scheduler'
|
||||
SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO
|
||||
|
||||
#HTTPCACHE_DIR = '/tmp/cache2' # if set, enables HTTP cache
|
||||
#HTTPCACHE_IGNORE_MISSING = 0 # ignore requests not in cache
|
||||
#HTTPCACHE_SECTORIZE = 1 # sectorize domains to distribute storage among servers
|
||||
|
||||
#STATS_ENABLED = 1 # enable stats
|
||||
#STATS_CLEANUP = 0 # cleanup domain stats when a domain is closed (saves memory)
|
||||
#STATS_DEBUG = 0 # log stats on domain closed
|
||||
|
||||
EXTENSIONS = (
|
||||
'scrapy.management.web.WebConsole',
|
||||
'scrapy.management.telnet.TelnetConsole',
|
||||
)
|
||||
|
||||
ITEM_PIPELINES = (
|
||||
'googledir.pipelines.GoogledirPipeline',
|
||||
)
|
||||
|
||||
#DEPTH_LIMIT = 10 # limit the maximum link depth to follow
|
||||
#DEPTH_STATS = 1 # enable depth stats
|
||||
|
||||
# Limit URL length. See: http://www.boutell.com/newfaq/misc/urllength.html
|
||||
URLLENGTH_LIMIT = 2083
|
||||
|
||||
#WEBCONSOLE_ENABLED = 1
|
||||
#WEBCONSOLE_PORT = 8060 # if not set uses a dynamic port
|
||||
|
||||
#TELNETCONSOLE_ENABLED = 1
|
||||
#TELNETCONSOLE_PORT = 2020 # if not set uses a dynamic port
|
||||
|
||||
# global mail sending settings
|
||||
#MAIL_HOST = 'localhost'
|
||||
#MAIL_FROM = 'scrapybot@localhost'
|
||||
|
||||
# scrapy webservice
|
||||
WS_ENABLED = 0
|
||||
|
||||
SPIDERPROFILER_ENABLED = 0
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
# -*- coding: utf8 -*-
|
||||
import re
|
||||
|
||||
from scrapy.xpath import HtmlXPathSelector
|
||||
from scrapy.contrib.spiders import CrawlSpider, rule
|
||||
from scrapy.utils.misc import items_to_csv
|
||||
|
||||
from googledir.items import GoogledirItem, GoogledirItemAdaptor
|
||||
|
||||
class GoogleDirectorySpider(CrawlSpider):
|
||||
domain_name = 'google.com'
|
||||
start_urls = ['http://www.google.com/dirhp']
|
||||
|
||||
rules = (
|
||||
rule('google.com/[A-Z][a-zA-Z_/]+$', 'parse_category', follow=True),
|
||||
)
|
||||
csv_file = open('scraped_items.csv', 'ab+')
|
||||
|
||||
def parse_category(self, response):
|
||||
# The selector we're going to use in order to extract data from the page
|
||||
hxs = HtmlXPathSelector(response)
|
||||
|
||||
# The path to website links in directory page
|
||||
links = hxs.x('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font')
|
||||
|
||||
for link in links:
|
||||
extractor = GoogledirItemAdaptor()
|
||||
extractor.name = link.x('a/text()')
|
||||
extractor.url = link.x('a/@href')
|
||||
extractor.description = link.x('font[2]/text()')
|
||||
item = extractor.item_instance
|
||||
yield item
|
||||
|
||||
SPIDER = GoogleDirectorySpider()
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
from scrapy.spider import BaseSpider
|
||||
|
||||
class $classname(BaseSpider):
|
||||
domain_name = "$site"
|
||||
start_urls = (
|
||||
'http://www.$site/',
|
||||
)
|
||||
|
||||
def parse(self, response):
|
||||
return ()
|
||||
|
||||
SPIDER = $classname()
|
||||
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
# -*- coding: utf8 -*-
|
||||
import re
|
||||
|
||||
from scrapy.xpath import HtmlXPathSelector
|
||||
from scrapy.link.extractors import RegexLinkExtractor
|
||||
from scrapy.contrib.spiders import CrawlSpider, Rule
|
||||
from $project_name.items import ${ProjectName}Item
|
||||
|
||||
class $classname(CrawlSpider):
|
||||
domain_name = '$site'
|
||||
start_urls = ['http://www.$site/']
|
||||
|
||||
rules = (
|
||||
Rule(RegexLinkExtractor(allow=(r'Items/', )), 'parse_item', follow=True),
|
||||
)
|
||||
|
||||
def parse_item(self, response):
|
||||
i = ${ProjectName}Item()
|
||||
#xs = HtmlXPathSelector(response)
|
||||
#i.attribute('site_id', xs.x('//input[@id="sid"]/@value'))
|
||||
#i.attribute('name', xs.x('//div[@id="name"]'))
|
||||
#i.attribute('description', xs.x('//div[@id="description"]'))
|
||||
return [i]
|
||||
|
||||
SPIDER = $classname()
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
# -*- coding: utf8 -*-
|
||||
from scrapy.contrib.spiders import CSVFeedSpider
|
||||
from $project_name.items import ${ProjectName}Item
|
||||
|
||||
class $classname(CSVFeedSpider):
|
||||
domain_name = '$site'
|
||||
start_urls = ['http://www.$site/feed.csv']
|
||||
# headers = ['id', 'name', 'description', 'image_link']
|
||||
# delimiter = '\t'
|
||||
|
||||
# Do any adaptations you need here
|
||||
#def adapt_response(self, response):
|
||||
# return response
|
||||
|
||||
def parse_row(self, response, row):
|
||||
i = ${ProjectName}Item()
|
||||
#i.attribute('url', row['url'])
|
||||
#i.attribute('name', row['name'])
|
||||
#i.attribute('description', row['description'])
|
||||
return i
|
||||
|
||||
SPIDER = $classname()
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
# -*- coding: utf8 -*-
|
||||
from scrapy.contrib.spiders import XMLFeedSpider
|
||||
from $project_name.items import ${ProjectName}Item
|
||||
|
||||
class $classname(XMLFeedSpider):
|
||||
domain_name = '$site'
|
||||
start_urls = ['http://www.$site/feed.xml']
|
||||
|
||||
def parse_item(self, response, xSel):
|
||||
i = ${ProjectName}Item()
|
||||
#i.attribute('url', xSel('url'))
|
||||
#i.attribute('name', xSel('name'))
|
||||
#i.attribute('description', xSel('description'))
|
||||
return i
|
||||
|
||||
SPIDER = $classname()
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import os
|
||||
os.environ.setdefault('SCRAPYSETTINGS_MODULE', 'googledir.settings')
|
||||
|
||||
from scrapy.command.cmdline import execute
|
||||
execute()
|
||||
Loading…
Reference in New Issue