From 04bd65de003e5ddaf49fa84e24993dec80bd5ced Mon Sep 17 00:00:00 2001 From: elpolilla Date: Wed, 26 Nov 2008 12:49:14 +0000 Subject: [PATCH] Added new crawling spider (with rules), but without replacing the previous one --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40429 --- scrapy/trunk/scrapy/contrib/item/__init__.py | 2 +- scrapy/trunk/scrapy/contrib/item/models.py | 5 - scrapy/trunk/scrapy/contrib/spiders2.py | 83 ----------- .../trunk/scrapy/contrib/spiders2/__init__.py | 2 + scrapy/trunk/scrapy/contrib/spiders2/crawl.py | 140 ++++++++++++++++++ scrapy/trunk/scrapy/contrib/spiders2/feed.py | 79 ++++++++++ scrapy/trunk/scrapy/item/models.py | 3 +- 7 files changed, 224 insertions(+), 90 deletions(-) delete mode 100644 scrapy/trunk/scrapy/contrib/spiders2.py create mode 100644 scrapy/trunk/scrapy/contrib/spiders2/__init__.py create mode 100644 scrapy/trunk/scrapy/contrib/spiders2/crawl.py create mode 100644 scrapy/trunk/scrapy/contrib/spiders2/feed.py diff --git a/scrapy/trunk/scrapy/contrib/item/__init__.py b/scrapy/trunk/scrapy/contrib/item/__init__.py index 8f029a687..a6226fec9 100644 --- a/scrapy/trunk/scrapy/contrib/item/__init__.py +++ b/scrapy/trunk/scrapy/contrib/item/__init__.py @@ -1 +1 @@ -from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline, SetGUIDPipeline +from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline diff --git a/scrapy/trunk/scrapy/contrib/item/models.py b/scrapy/trunk/scrapy/contrib/item/models.py index 65ff653de..d7fe8e695 100644 --- a/scrapy/trunk/scrapy/contrib/item/models.py +++ b/scrapy/trunk/scrapy/contrib/item/models.py @@ -30,11 +30,6 @@ class ValidationPipeline(object): item.validate() return item -class SetGUIDPipeline(object): - def process_item(self, domain, response, item): - spiders.fromdomain(domain).set_guid(item) - return item - class RobustScrapedItem(ScrapedItem): """ A more robust scraped item class with a built-in validation mechanism and diff --git a/scrapy/trunk/scrapy/contrib/spiders2.py b/scrapy/trunk/scrapy/contrib/spiders2.py deleted file mode 100644 index e8f7117ad..000000000 --- a/scrapy/trunk/scrapy/contrib/spiders2.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -This module contains the basic crawling spider that you can use to inherit your -spider from. -""" - -from scrapy.http import Request -from scrapy.spider import BaseSpider -from scrapy.item import ScrapedItem - -class CrawlSpider(BaseSpider): - """ - This is the base class for crawling spiders. It is based on a list of - crawling rules (stored in the "rules" attribute) which specify how links - are extracted and followed, and how pages are processed (by using a - callback function). - - For more info about rules see the Rule class. - """ - - def parse(self, response): - """Method called by the framework core for all the start_urls. Do not - override this function, override parse_start_url instead. - """ - if response.url in self.start_urls: - return self._parse_wrapper(response, self.parse_start_url, cb_kwargs={}, follow=True) - else: - return self.parse_url(response) - - def parse_start_url(self, response): - """Callback function for processing start_urls. It must return a list - of ScrapedItems and/or Requests. - """ - return [] - - def _requests_to_follow(self, response): - requests = [] - seen = set() - for rule in self.rules: - callback = rule.callback if callable(rule.callback) else getattr(self, rule.callback, None) - links = [l for l in rule.link_extractor.extract_urls(response) if l not in seen] - seen.union(links) - for link in links: - r = Request(url=link.url, link_text=link.text) - r.append_callback(self._parse_wrapper, callback, cb_kwargs=rule.cb_kwargs, follow=rule.follow) - requests.append(r) - return requests - - def _parse_wrapper(self, response, callback, cb_kwargs, follow): - res = [] - if follow: - res.extend(self._requests_to_follow(response)) - if callback: - res.extend(callback(response, **cb_kwargs) or ()) - return res - - -class Rule(object): - """ - A rule for crawling, which receives the following constructor arguments: - - link_extractor (required) - A LinkExtractor which defines the policy for extracting links - callback (optional) - A function to use to process the page once it has been downloaded. If - callback is omitted the page is not procesed, just crawled. If callback - is a string (instead of callable) a method of the spider class with that - name is used as the callback function - cb_kwargs (optional) - A dict specifying keyword arguments to pass to the callback function - follow (optional) - If True, links will be followed from the pages crawled by this rule. - It defaults to True when no callback is specified or False when a - callback is specified - """ - - def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None): - self.link_extractor = link_extractor - self.callback = callback - self.cb_kwargs = cb_kwargs or {} - if follow is None: - self.follow = False if callback else True - else: - self.follow = follow diff --git a/scrapy/trunk/scrapy/contrib/spiders2/__init__.py b/scrapy/trunk/scrapy/contrib/spiders2/__init__.py new file mode 100644 index 000000000..1b7107f45 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/spiders2/__init__.py @@ -0,0 +1,2 @@ +from scrapy.contrib.spiders2.crawl import CrawlSpider, Rule +from scrapy.contrib.spiders2.feed import XMLFeedSpider, CSVFeedSpider diff --git a/scrapy/trunk/scrapy/contrib/spiders2/crawl.py b/scrapy/trunk/scrapy/contrib/spiders2/crawl.py new file mode 100644 index 000000000..c56a740f2 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/spiders2/crawl.py @@ -0,0 +1,140 @@ +from scrapy.http import Request +from scrapy.spider import BaseSpider +from scrapy.item import ScrapedItem + +class Rule(object): + """ + A rule for crawling, which receives the following constructor arguments: + + link_extractor (required) + A LinkExtractor which defines the policy for extracting links + callback (optional) + A function to use to process the page once it has been downloaded. If + callback is omitted the page is not procesed, just crawled. If callback + is a string (instead of callable) a method of the spider class with that + name is used as the callback function + cb_kwargs (optional) + A dict specifying keyword arguments to pass to the callback function + follow (optional) + If True, links will be followed from the pages crawled by this rule. + It defaults to True when no callback is specified or False when a + callback is specified + link_filter (optional) + Can be either a callable, or a string with the name of a method defined + in the spider's class. + This method will be called with the list of extracted links matching + this rule (if any) and must return another list of links. + """ + + def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, link_filter=None): + self.link_extractor = link_extractor + self.callback = callback + self.cb_kwargs = cb_kwargs or {} + self.link_filter = link_filter + if follow is None: + self.follow = False if callback else True + else: + self.follow = follow + +class CrawlSpider(BaseSpider): + """ + Class for spiders that crawl over web pages and extract/parse their links + given some crawling rules. + + These crawling rules are established by setting the 'rules' class attribute, + which is a tuple of Rule objects. + When the spider is running, it iterates over these rules with each response + and do what it has to (extract links if follow=True, and return items/requests if + there's a parsing method defined in the rule). + """ + rules = () + + def __init__(self): + def _get_method(method): + if isinstance(method, basestring): + return getattr(self, method, None) + elif method and callable(method): + return method + + super(CrawlSpider, self).__init__() + if not hasattr(self, 'rules'): + return + for rule in self.rules: + rule.callback = _get_method(rule.callback) + rule.link_filter = _get_method(rule.link_filter) + + def parse(self, response): + """This function is called by the core for all the start_urls. Do not + override this function, override parse_start_url instead.""" + if response.url in self.start_urls: + return self._parse_wrapper(response, self.parse_start_url, cb_kwargs={}, follow=True) + else: + return self.parse_url(response) + + def parse_start_url(self, response): + """Callback function for processing start_urls. It must return a list + of ScrapedItems and/or Requests.""" + return [] + + def scraped_item(self, response, item): + """ + This method is called for each item returned by the spider, and it's intended + to do anything that it's needed before returning the item to the core, specially + setting its GUID. + It receives and returns an item + """ + return item + + def _requests_to_follow(self, response): + """ + This method iterates over each of the spider's rules, extracts the links + matching each case, filters them (if needed), and returns a list of unique + requests per response. + """ + requests = [] + seen = set() + for rule in self.rules: + links = [link for link in rule.link_extractor.extract_urls(response) if link not in seen] + if rule.link_filter: + links = rule.link_filter(links) + seen = seen.union(links) + for link in links: + r = Request(url=link.url, link_text=link.text) + r.append_callback(self._parse_wrapper, rule.callback, cb_kwargs=rule.cb_kwargs, follow=rule.follow) + requests.append(r) + return requests + + def _parse_wrapper(self, response, callback, cb_kwargs, follow): + """ + This is were any response (except the ones from the start urls) arrives, and + were it's decided whether to extract links or not from it, and if it will + be parsed or not. + It returns a list of requests/items. + """ + res = [] + if follow: + res.extend(self._requests_to_follow(response)) + if callback: + cb_res = callback(response, **cb_kwargs) or () + for entry in cb_res: + if isinstance(entry, ScrapedItem): + entry = self.scraped_item(response, entry) + res.extend(cb_res) + return res + + def parse_url(self, response): + """ + This method is called whenever you run scrapy with the 'parse' command + over an URL. + """ + ret = set() + for rule in self.rules: + links = [link for link in rule.link_extractor.extract_urls(response) if link not in ret] + if rule.link_filter: + links = rule.link_filter(links) + ret = ret.union(links) + + if rule.callback and rule.link_extractor.match(response.url): + ret = ret.union(rule.callback(response)) + return list(ret) + diff --git a/scrapy/trunk/scrapy/contrib/spiders2/feed.py b/scrapy/trunk/scrapy/contrib/spiders2/feed.py new file mode 100644 index 000000000..9ebefb2e0 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/spiders2/feed.py @@ -0,0 +1,79 @@ +from scrapy.spider import BaseSpider +from scrapy.item import ScrapedItem +from scrapy.utils.iterators import xmliter, csviter +from scrapy.xpath.selector import XmlXPathSelector +from scrapy.core.exceptions import NotConfigured + +class XMLFeedSpider(BaseSpider): + """ + This class intends to be the base class for spiders that scrape + from XML feeds. + + You can choose whether to parse the file using the iternodes tool, + or not using it (which just splits the tags using xpath) + """ + iternodes = True + itertag = 'item' + + def item_scraped(self, response, item): + """ + This method is called for each item returned by the spider, and it's intended + to do anything that it's needed before returning the item to the core, specially + setting its GUID. + It receives and returns an item + """ + return item + + def parse_item_wrapper(self, response, xSel): + ret = self.parse_item(response, xSel) + if isinstance(ret, ScrapedItem): + self.scraped_item(response, ret) + return ret + + def parse(self, response): + if not hasattr(self, 'parse_item'): + raise NotConfigured('You must define parse_item method in order to scrape this XML feed') + + if self.iternodes: + nodes = xmliter(response, self.itertag) + else: + nodes = XmlXPathSelector(response).x('//%s' % self.itertag) + + return (self.parse_item_wrapper(response, xSel) for xSel in nodes) + +class CSVFeedSpider(BaseSpider): + """ + Spider for parsing CSV feeds. + It receives a CSV file in a response; iterates through each of its rows, + and calls parse_row with a dict containing each field's data. + + You can set some options regarding the CSV file, such as the delimiter + and the file's headers. + """ + delimiter = None # When this is None, python's csv module's default delimiter is used + headers = None + + def scraped_item(self, response, item): + """This method has the same purpose as the one in XMLFeedSpider""" + return item + + def adapt_feed(self, response): + """You can override this function in order to make any changes you want + to into the feed before parsing it. This function may return either a + response or a string. """ + + return response + + def parse_row_wrapper(self, response, row): + ret = self.parse_row(response, row) + if isinstance(ret, ScrapedItem): + self.scraped_item(response, ret) + return ret + + def parse(self, response): + if not hasattr(self, 'parse_row'): + raise NotConfigured('You must define parse_row method in order to scrape this CSV feed') + + feed = self.adapt_feed(response) + return (self.parse_row_wrapper(feed, row) for row in csviter(response, self.delimiter, self.headers)) + diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py index 5ab484062..abe3861a3 100644 --- a/scrapy/trunk/scrapy/item/models.py +++ b/scrapy/trunk/scrapy/item/models.py @@ -1,5 +1,5 @@ -from scrapy.item.adaptors import AdaptorPipe from scrapy.conf import settings +from scrapy.item.adaptors import AdaptorPipe class ScrapedItem(object): """ @@ -7,6 +7,7 @@ class ScrapedItem(object): 'guid' attribute is required, and is an attribute that identifies uniquely the given scraped item. """ + _adaptors_dict = {} def set_adaptors(self, adaptors_dict): """