RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added new crawling spider (with rules), but without replacing the previous one 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40429
This commit is contained in:
elpolilla 2008-11-26 12:49:14 +00:00
parent fe49bc2011
commit 04bd65de00
7 changed files with 224 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/trunk/scrapy/contrib/item/__init__.py
View File

@ -1 +1 @@
from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline, SetGUIDPipeline
from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline

5
scrapy/trunk/scrapy/contrib/item/models.py
View File

@ -30,11 +30,6 @@ class ValidationPipeline(object):
item.validate()
return item
class SetGUIDPipeline(object):
def process_item(self, domain, response, item):
spiders.fromdomain(domain).set_guid(item)
return item
class RobustScrapedItem(ScrapedItem):
"""
A more robust scraped item class with a built-in validation mechanism and

83
scrapy/trunk/scrapy/contrib/spiders2.py
View File

@ -1,83 +0,0 @@
"""
This module contains the basic crawling spider that you can use to inherit your
spider from.
"""
from scrapy.http import Request
from scrapy.spider import BaseSpider
from scrapy.item import ScrapedItem
class CrawlSpider(BaseSpider):
"""
This is the base class for crawling spiders. It is based on a list of
crawling rules (stored in the "rules" attribute) which specify how links
are extracted and followed, and how pages are processed (by using a
callback function).
For more info about rules see the Rule class.
"""
def parse(self, response):
"""Method called by the framework core for all the start_urls. Do not
override this function, override parse_start_url instead.
"""
if response.url in self.start_urls:
return self._parse_wrapper(response, self.parse_start_url, cb_kwargs={}, follow=True)
else:
return self.parse_url(response)
def parse_start_url(self, response):
"""Callback function for processing start_urls. It must return a list
of ScrapedItems and/or Requests.
"""
return []
def _requests_to_follow(self, response):
requests = []
seen = set()
for rule in self.rules:
callback = rule.callback if callable(rule.callback) else getattr(self, rule.callback, None)
links = [l for l in rule.link_extractor.extract_urls(response) if l not in seen]
seen.union(links)
for link in links:
r = Request(url=link.url, link_text=link.text)
r.append_callback(self._parse_wrapper, callback, cb_kwargs=rule.cb_kwargs, follow=rule.follow)
requests.append(r)
return requests
def _parse_wrapper(self, response, callback, cb_kwargs, follow):
res = []
if follow:
res.extend(self._requests_to_follow(response))
if callback:
res.extend(callback(response, **cb_kwargs) or ())
return res
class Rule(object):
"""
A rule for crawling, which receives the following constructor arguments:
link_extractor (required)
A LinkExtractor which defines the policy for extracting links
callback (optional)
A function to use to process the page once it has been downloaded. If
callback is omitted the page is not procesed, just crawled. If callback
is a string (instead of callable) a method of the spider class with that
name is used as the callback function
cb_kwargs (optional)
A dict specifying keyword arguments to pass to the callback function
follow (optional)
If True, links will be followed from the pages crawled by this rule.
It defaults to True when no callback is specified or False when a
callback is specified
"""
def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None):
self.link_extractor = link_extractor
self.callback = callback
self.cb_kwargs = cb_kwargs or {}
if follow is None:
self.follow = False if callback else True
else:
self.follow = follow

2
scrapy/trunk/scrapy/contrib/spiders2/__init__.py Normal file
View File

@ -0,0 +1,2 @@
from scrapy.contrib.spiders2.crawl import CrawlSpider, Rule
from scrapy.contrib.spiders2.feed import XMLFeedSpider, CSVFeedSpider

140
scrapy/trunk/scrapy/contrib/spiders2/crawl.py Normal file
View File

@ -0,0 +1,140 @@
from scrapy.http import Request
from scrapy.spider import BaseSpider
from scrapy.item import ScrapedItem
class Rule(object):
"""
A rule for crawling, which receives the following constructor arguments:
link_extractor (required)
A LinkExtractor which defines the policy for extracting links
callback (optional)
A function to use to process the page once it has been downloaded. If
callback is omitted the page is not procesed, just crawled. If callback
is a string (instead of callable) a method of the spider class with that
name is used as the callback function
cb_kwargs (optional)
A dict specifying keyword arguments to pass to the callback function
follow (optional)
If True, links will be followed from the pages crawled by this rule.
It defaults to True when no callback is specified or False when a
callback is specified
link_filter (optional)
Can be either a callable, or a string with the name of a method defined
in the spider's class.
This method will be called with the list of extracted links matching
this rule (if any) and must return another list of links.
"""
def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, link_filter=None):
self.link_extractor = link_extractor
self.callback = callback
self.cb_kwargs = cb_kwargs or {}
self.link_filter = link_filter
if follow is None:
self.follow = False if callback else True
else:
self.follow = follow
class CrawlSpider(BaseSpider):
"""
Class for spiders that crawl over web pages and extract/parse their links
given some crawling rules.
These crawling rules are established by setting the 'rules' class attribute,
which is a tuple of Rule objects.
When the spider is running, it iterates over these rules with each response
and do what it has to (extract links if follow=True, and return items/requests if
there's a parsing method defined in the rule).
"""
rules = ()
def __init__(self):
def _get_method(method):
if isinstance(method, basestring):
return getattr(self, method, None)
elif method and callable(method):
return method
super(CrawlSpider, self).__init__()
if not hasattr(self, 'rules'):
return
for rule in self.rules:
rule.callback = _get_method(rule.callback)
rule.link_filter = _get_method(rule.link_filter)
def parse(self, response):
"""This function is called by the core for all the start_urls. Do not
override this function, override parse_start_url instead."""
if response.url in self.start_urls:
return self._parse_wrapper(response, self.parse_start_url, cb_kwargs={}, follow=True)
else:
return self.parse_url(response)
def parse_start_url(self, response):
"""Callback function for processing start_urls. It must return a list
of ScrapedItems and/or Requests."""
return []
def scraped_item(self, response, item):
"""
This method is called for each item returned by the spider, and it's intended
to do anything that it's needed before returning the item to the core, specially
setting its GUID.
It receives and returns an item
"""
return item
def _requests_to_follow(self, response):
"""
This method iterates over each of the spider's rules, extracts the links
matching each case, filters them (if needed), and returns a list of unique
requests per response.
"""
requests = []
seen = set()
for rule in self.rules:
links = [link for link in rule.link_extractor.extract_urls(response) if link not in seen]
if rule.link_filter:
links = rule.link_filter(links)
seen = seen.union(links)
for link in links:
r = Request(url=link.url, link_text=link.text)
r.append_callback(self._parse_wrapper, rule.callback, cb_kwargs=rule.cb_kwargs, follow=rule.follow)
requests.append(r)
return requests
def _parse_wrapper(self, response, callback, cb_kwargs, follow):
"""
This is were any response (except the ones from the start urls) arrives, and
were it's decided whether to extract links or not from it, and if it will
be parsed or not.
It returns a list of requests/items.
"""
res = []
if follow:
res.extend(self._requests_to_follow(response))
if callback:
cb_res = callback(response, **cb_kwargs) or ()
for entry in cb_res:
if isinstance(entry, ScrapedItem):
entry = self.scraped_item(response, entry)
res.extend(cb_res)
return res
def parse_url(self, response):
"""
This method is called whenever you run scrapy with the 'parse' command
over an URL.
"""
ret = set()
for rule in self.rules:
links = [link for link in rule.link_extractor.extract_urls(response) if link not in ret]
if rule.link_filter:
links = rule.link_filter(links)
ret = ret.union(links)
if rule.callback and rule.link_extractor.match(response.url):
ret = ret.union(rule.callback(response))
return list(ret)

79
scrapy/trunk/scrapy/contrib/spiders2/feed.py Normal file
View File

@ -0,0 +1,79 @@
from scrapy.spider import BaseSpider
from scrapy.item import ScrapedItem
from scrapy.utils.iterators import xmliter, csviter
from scrapy.xpath.selector import XmlXPathSelector
from scrapy.core.exceptions import NotConfigured
class XMLFeedSpider(BaseSpider):
"""
This class intends to be the base class for spiders that scrape
from XML feeds.
You can choose whether to parse the file using the iternodes tool,
or not using it (which just splits the tags using xpath)
"""
iternodes = True
itertag = 'item'
def item_scraped(self, response, item):
"""
This method is called for each item returned by the spider, and it's intended
to do anything that it's needed before returning the item to the core, specially
setting its GUID.
It receives and returns an item
"""
return item
def parse_item_wrapper(self, response, xSel):
ret = self.parse_item(response, xSel)
if isinstance(ret, ScrapedItem):
self.scraped_item(response, ret)
return ret
def parse(self, response):
if not hasattr(self, 'parse_item'):
raise NotConfigured('You must define parse_item method in order to scrape this XML feed')
if self.iternodes:
nodes = xmliter(response, self.itertag)
else:
nodes = XmlXPathSelector(response).x('//%s' % self.itertag)
return (self.parse_item_wrapper(response, xSel) for xSel in nodes)
class CSVFeedSpider(BaseSpider):
"""
Spider for parsing CSV feeds.
It receives a CSV file in a response; iterates through each of its rows,
and calls parse_row with a dict containing each field's data.
You can set some options regarding the CSV file, such as the delimiter
and the file's headers.
"""
delimiter = None # When this is None, python's csv module's default delimiter is used
headers = None
def scraped_item(self, response, item):
"""This method has the same purpose as the one in XMLFeedSpider"""
return item
def adapt_feed(self, response):
"""You can override this function in order to make any changes you want
to into the feed before parsing it. This function may return either a
response or a string. """
return response
def parse_row_wrapper(self, response, row):
ret = self.parse_row(response, row)
if isinstance(ret, ScrapedItem):
self.scraped_item(response, ret)
return ret
def parse(self, response):
if not hasattr(self, 'parse_row'):
raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
feed = self.adapt_feed(response)
return (self.parse_row_wrapper(feed, row) for row in csviter(response, self.delimiter, self.headers))

3
scrapy/trunk/scrapy/item/models.py
View File

@ -1,5 +1,5 @@
from scrapy.item.adaptors import AdaptorPipe
from scrapy.conf import settings
from scrapy.item.adaptors import AdaptorPipe
class ScrapedItem(object):
"""
@ -7,6 +7,7 @@ class ScrapedItem(object):
'guid' attribute is required, and is an attribute
that identifies uniquely the given scraped item.
"""
_adaptors_dict = {}
def set_adaptors(self, adaptors_dict):
"""