mirror of https://github.com/scrapy/scrapy.git
removed experimental examples
This commit is contained in:
Pablo Hoffman
parent
bb2b67c862
commit
cf572bb642
|
|
@ -1 +0,0 @@
|
|||
# googledir project
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
# Define here the models for your scraped items
|
||||
#
|
||||
# See documentation in:
|
||||
# http://doc.scrapy.org/topics/items.html
|
||||
|
||||
from scrapy.item import Item, Field
|
||||
|
||||
class GoogledirItem(Item):
|
||||
|
||||
name = Field(default='')
|
||||
url = Field(default='')
|
||||
description = Field(default='')
|
||||
|
||||
def __str__(self):
|
||||
return "Google Category: name=%s url=%s" \
|
||||
% (self['name'], self['url'])
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
# Define your item pipelines here
|
||||
#
|
||||
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
||||
# See: http://doc.scrapy.org/topics/item-pipeline.html
|
||||
|
||||
from scrapy.exceptions import DropItem
|
||||
|
||||
class FilterWordsPipeline(object):
|
||||
"""
|
||||
A pipeline for filtering out items which contain certain
|
||||
words in their description
|
||||
"""
|
||||
|
||||
# put all words in lowercase
|
||||
words_to_filter = ['politics', 'religion']
|
||||
|
||||
def process_item(self, item, spider):
|
||||
for word in self.words_to_filter:
|
||||
if word in unicode(item['description']).lower():
|
||||
raise DropItem("Contains forbidden word: %s" % word)
|
||||
else:
|
||||
return item
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
# Scrapy settings for googledir project
|
||||
#
|
||||
# For simplicity, this file contains only the most important settings by
|
||||
# default. All the other settings are documented here:
|
||||
#
|
||||
# http://doc.scrapy.org/topics/settings.html
|
||||
|
||||
BOT_NAME = 'googledir'
|
||||
BOT_VERSION = '1.0'
|
||||
|
||||
SPIDER_MODULES = ['googledir.spiders']
|
||||
NEWSPIDER_MODULE = 'googledir.spiders'
|
||||
DEFAULT_ITEM_CLASS = 'googledir.items.GoogledirItem'
|
||||
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
||||
|
||||
ITEM_PIPELINES = ['googledir.pipelines.FilterWordsPipeline']
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
# This package will contain the spiders of your Scrapy project
|
||||
#
|
||||
# To create the first spider for your project use this command:
|
||||
#
|
||||
# scrapy genspider myspider myspider-domain.com
|
||||
#
|
||||
# For more info see:
|
||||
# http://doc.scrapy.org/topics/spiders.html
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
from scrapy.selector import HtmlXPathSelector
|
||||
from scrapy.contrib.loader import XPathItemLoader
|
||||
from scrapy.contrib_exp.crawlspider import CrawlSpider, Rule
|
||||
|
||||
from googledir.items import GoogledirItem
|
||||
|
||||
class GoogleDirectorySpider(CrawlSpider):
|
||||
|
||||
name = 'google_directory'
|
||||
allowed_domains = ['directory.google.com']
|
||||
start_urls = ['http://directory.google.com/']
|
||||
|
||||
rules = (
|
||||
# search for categories pattern and follow links
|
||||
Rule(r'/[A-Z][a-zA-Z_/]+$', 'parse_category', follow=True),
|
||||
)
|
||||
|
||||
def parse_category(self, response):
|
||||
# The main selector we're using to extract data from the page
|
||||
main_selector = HtmlXPathSelector(response)
|
||||
|
||||
# The XPath to website links in the directory page
|
||||
xpath = '//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font'
|
||||
|
||||
# Get a list of (sub) selectors to each website node pointed by the XPath
|
||||
sub_selectors = main_selector.select(xpath)
|
||||
|
||||
# Iterate over the sub-selectors to extract data for each website
|
||||
for selector in sub_selectors:
|
||||
item = GoogledirItem()
|
||||
|
||||
l = XPathItemLoader(item=item, selector=selector)
|
||||
l.add_xpath('name', 'a/text()')
|
||||
l.add_xpath('url', 'a/@href')
|
||||
l.add_xpath('description', 'font[2]/text()')
|
||||
|
||||
# Here we populate the item and yield it
|
||||
yield l.load_item()
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
[settings]
|
||||
default = googledir.settings
|
||||
|
|
@ -1 +0,0 @@
|
|||
# package
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
# Define here the models for your scraped items
|
||||
#
|
||||
# See documentation in:
|
||||
# http://doc.scrapy.org/topics/items.html
|
||||
|
||||
from scrapy.item import Item, Field
|
||||
|
||||
class ImdbItem(Item):
|
||||
# define the fields for your item here like:
|
||||
# name = Field()
|
||||
title = Field()
|
||||
url = Field()
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
# Define your item pipelines here
|
||||
#
|
||||
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
||||
# See: http://doc.scrapy.org/topics/item-pipeline.html
|
||||
|
||||
class ImdbPipeline(object):
|
||||
def process_item(self, item, spider):
|
||||
return item
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
# Scrapy settings for imdb project
|
||||
#
|
||||
# For simplicity, this file contains only the most important settings by
|
||||
# default. All the other settings are documented here:
|
||||
#
|
||||
# http://doc.scrapy.org/topics/settings.html
|
||||
|
||||
BOT_NAME = 'imdb'
|
||||
BOT_VERSION = '1.0'
|
||||
|
||||
SPIDER_MODULES = ['imdb.spiders']
|
||||
NEWSPIDER_MODULE = 'imdb.spiders'
|
||||
DEFAULT_ITEM_CLASS = 'imdb.items.ImdbItem'
|
||||
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
||||
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
# This package will contain the spiders of your Scrapy project
|
||||
#
|
||||
# To create the first spider for your project use this command:
|
||||
#
|
||||
# scrapy genspider myspider myspider-domain.com
|
||||
#
|
||||
# For more info see:
|
||||
# http://doc.scrapy.org/topics/spiders.html
|
||||
|
|
@ -1,139 +0,0 @@
|
|||
from scrapy.http import Request
|
||||
from scrapy.selector import HtmlXPathSelector
|
||||
from scrapy.contrib.loader import XPathItemLoader
|
||||
from scrapy.contrib_exp.crawlspider import CrawlSpider, Rule
|
||||
from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor
|
||||
from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize, \
|
||||
FilterDupes, FilterUrl
|
||||
from scrapy.utils.url import urljoin_rfc
|
||||
|
||||
from imdb.items import ImdbItem, Field
|
||||
|
||||
from itertools import chain, imap, izip
|
||||
|
||||
class UsaOpeningWeekMovie(ImdbItem):
|
||||
pass
|
||||
|
||||
class UsaTopWeekMovie(ImdbItem):
|
||||
pass
|
||||
|
||||
class Top250Movie(ImdbItem):
|
||||
rank = Field()
|
||||
rating = Field()
|
||||
year = Field()
|
||||
votes = Field()
|
||||
|
||||
class MovieItem(ImdbItem):
|
||||
release_date = Field()
|
||||
tagline = Field()
|
||||
|
||||
|
||||
class ImdbSiteSpider(CrawlSpider):
|
||||
name = 'imdb.com'
|
||||
allowed_domains = ['imdb.com']
|
||||
start_urls = ['http://www.imdb.com/']
|
||||
|
||||
# extract requests using this classes from urls matching 'follow' flag
|
||||
request_extractors = [
|
||||
SgmlRequestExtractor(tags=['a'], attrs=['href']),
|
||||
]
|
||||
|
||||
# process requests using this classes from urls matching 'follow' flag
|
||||
request_processors = [
|
||||
Canonicalize(),
|
||||
FilterDupes(),
|
||||
FilterUrl(deny=r'/tt\d+/$'), # deny movie url as we will dispatch
|
||||
# manually the movie requests
|
||||
]
|
||||
|
||||
# include domain bit for demo purposes
|
||||
rules = (
|
||||
# these two rules expects requests from start url
|
||||
Rule(r'imdb.com/nowplaying/$', 'parse_now_playing'),
|
||||
Rule(r'imdb.com/chart/top$', 'parse_top_250'),
|
||||
# this rule will parse requests manually dispatched
|
||||
Rule(r'imdb.com/title/tt\d+/$', 'parse_movie_info'),
|
||||
)
|
||||
|
||||
def parse_now_playing(self, response):
|
||||
"""Scrapes USA openings this week and top 10 in week"""
|
||||
self.log("Parsing USA Top Week")
|
||||
hxs = HtmlXPathSelector(response)
|
||||
|
||||
_urljoin = lambda url: self._urljoin(response, url)
|
||||
|
||||
#
|
||||
# openings this week
|
||||
#
|
||||
openings = hxs.select('//table[@class="movies"]//a[@class="title"]')
|
||||
boxoffice = hxs.select('//table[@class="boxoffice movies"]//a[@class="title"]')
|
||||
|
||||
opening_titles = openings.select('text()').extract()
|
||||
opening_urls = imap(_urljoin, openings.select('@href').extract())
|
||||
|
||||
box_titles = boxoffice.select('text()').extract()
|
||||
box_urls = imap(_urljoin, boxoffice.select('@href').extract())
|
||||
|
||||
# items
|
||||
opening_items = (UsaOpeningWeekMovie(title=title, url=url)
|
||||
for (title, url)
|
||||
in izip(opening_titles, opening_urls))
|
||||
|
||||
box_items = (UsaTopWeekMovie(title=title, url=url)
|
||||
for (title, url)
|
||||
in izip(box_titles, box_urls))
|
||||
|
||||
# movie requests
|
||||
requests = imap(self.make_requests_from_url,
|
||||
chain(opening_urls, box_urls))
|
||||
|
||||
return chain(opening_items, box_items, requests)
|
||||
|
||||
def parse_top_250(self, response):
|
||||
"""Scrapes movies from top 250 list"""
|
||||
self.log("Parsing Top 250")
|
||||
hxs = HtmlXPathSelector(response)
|
||||
|
||||
# scrap each row in the table
|
||||
rows = hxs.select('//div[@id="main"]/table/tr//a/ancestor::tr')
|
||||
for row in rows:
|
||||
fields = row.select('td//text()').extract()
|
||||
url, = row.select('td//a/@href').extract()
|
||||
url = self._urljoin(response, url)
|
||||
|
||||
item = Top250Movie()
|
||||
item['title'] = fields[2]
|
||||
item['url'] = url
|
||||
item['rank'] = fields[0]
|
||||
item['rating'] = fields[1]
|
||||
item['year'] = fields[3]
|
||||
item['votes'] = fields[4]
|
||||
|
||||
# scrapped top250 item
|
||||
yield item
|
||||
# fetch movie
|
||||
yield self.make_requests_from_url(url)
|
||||
|
||||
def parse_movie_info(self, response):
|
||||
"""Scrapes movie information"""
|
||||
self.log("Parsing Movie Info")
|
||||
hxs = HtmlXPathSelector(response)
|
||||
selector = hxs.select('//div[@class="maindetails"]')
|
||||
|
||||
item = MovieItem()
|
||||
# set url
|
||||
item['url'] = response.url
|
||||
|
||||
# use item loader for other attributes
|
||||
l = XPathItemLoader(item=item, selector=selector)
|
||||
l.add_xpath('title', './/h1/text()')
|
||||
l.add_xpath('release_date', './/h5[text()="Release Date:"]'
|
||||
'/following-sibling::div/text()')
|
||||
l.add_xpath('tagline', './/h5[text()="Tagline:"]'
|
||||
'/following-sibling::div/text()')
|
||||
|
||||
yield l.load_item()
|
||||
|
||||
def _urljoin(self, response, url):
|
||||
"""Helper to convert relative urls to absolute"""
|
||||
return urljoin_rfc(response.url, url, response.encoding)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
[settings]
|
||||
default = imdb.settings
|
||||
Loading…
Reference in New Issue