From 7afc9157177cb84bd2cdbabe4456a22e902ece09 Mon Sep 17 00:00:00 2001 From: Ismael Carnales Date: Tue, 21 Jul 2009 16:11:21 -0300 Subject: [PATCH] added BaseItem as base item class, and moved extra functionality of ScrapedItem to RobustScrapedItem --- scrapy/command/commands/parse.py | 5 +++-- scrapy/contrib/item/models.py | 9 +++++++-- scrapy/contrib/spiders/crawl.py | 2 +- scrapy/contrib/spiders/feed.py | 8 ++++---- scrapy/contrib/web/service.py | 4 ++-- scrapy/contrib_exp/newitem/models.py | 4 ++-- scrapy/core/scraper.py | 6 +++--- scrapy/item/__init__.py | 2 +- scrapy/item/models.py | 19 +++++-------------- scrapy/item/pipeline.py | 6 +++--- scrapy/utils/display.py | 4 ++-- 11 files changed, 33 insertions(+), 36 deletions(-) diff --git a/scrapy/command/commands/parse.py b/scrapy/command/commands/parse.py index 6e6e64d5d..4ee9912e2 100644 --- a/scrapy/command/commands/parse.py +++ b/scrapy/command/commands/parse.py @@ -1,7 +1,7 @@ from scrapy.command import ScrapyCommand from scrapy.fetcher import fetch from scrapy.http import Request -from scrapy.item import ScrapedItem +from scrapy.item.models import BaseItem from scrapy.spider import spiders from scrapy.utils import display from scrapy import log @@ -42,7 +42,8 @@ class Command(ScrapyCommand): result = callback_fcn(response) links = [i for i in result if isinstance(i, Request)] - items = [self.pipeline_process(i, spider, opts) for i in result if isinstance(i, ScrapedItem)] + items = [self.pipeline_process(i, spider, opts) for i in result if \ + isinstance(i, BaseItem)] return items, links return (), () diff --git a/scrapy/contrib/item/models.py b/scrapy/contrib/item/models.py index f172d808a..015e2ee0c 100644 --- a/scrapy/contrib/item/models.py +++ b/scrapy/contrib/item/models.py @@ -3,11 +3,12 @@ This module contains some extra base models for scraped items which could be useful in some Scrapy implementations """ +import copy import hashlib from pprint import PrettyPrinter -from scrapy.item import ScrapedItem, ItemDelta +from scrapy.item import ScrapedItem from scrapy.item.adaptors import AdaptorPipe from scrapy.core.exceptions import DropItem from scrapy.utils.python import unique @@ -194,6 +195,10 @@ class RobustScrapedItem(ScrapedItem): pipe.add_adaptor(adaptor, position) self.set_attrib_adaptors(attrib, pipe) + def copy(self): + """Create a new RobustScrapedItem based on the current one""" + return copy.deepcopy(self) + def validate(self): """Method used to validate item attributes data""" raise NotImplemented @@ -215,7 +220,7 @@ class RobustScrapedItem(ScrapedItem): return hash_.hexdigest() -class RobustItemDelta(ItemDelta): +class RobustItemDelta(object): """ This class represents the difference between a pair of RobustScrapedItems. diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py index c1ae74cf5..4f7115da8 100644 --- a/scrapy/contrib/spiders/crawl.py +++ b/scrapy/contrib/spiders/crawl.py @@ -71,7 +71,7 @@ class CrawlSpider(InitSpider): def parse_start_url(self, response): """Overrideable callback function for processing start_urls. It must - return a list of ScrapedItems and/or Requests""" + return a list of BaseItem and/or Requests""" return [] def process_results(self, response, results): diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py index 2ef9085c4..674d4e9c3 100644 --- a/scrapy/contrib/spiders/feed.py +++ b/scrapy/contrib/spiders/feed.py @@ -6,7 +6,7 @@ See documentation in docs/ref/spiders.rst """ from scrapy.contrib.spiders.init import InitSpider -from scrapy.item import ScrapedItem +from scrapy.item.models import BaseItem from scrapy.http import Request from scrapy.utils.iterators import xmliter, csviter from scrapy.xpath.selector import XmlXPathSelector, HtmlXPathSelector @@ -51,13 +51,13 @@ class XMLFeedSpider(InitSpider): """This method is called for the nodes matching the provided tag name (itertag). Receives the response and an XPathSelector for each node. Overriding this method is mandatory. Otherwise, you spider won't work. - This method must return either a ScrapedItem, a Request, or a list + This method must return either a BaseItem, a Request, or a list containing any of them. """ for xSel in nodes: ret = self.parse_item(response, xSel) - if isinstance(ret, (ScrapedItem, Request)): + if isinstance(ret, (BaseItem, Request)): ret = [ret] if not isinstance(ret, (list, tuple)): raise TypeError('You cannot return an "%s" object from a spider' % type(ret).__name__) @@ -121,7 +121,7 @@ class CSVFeedSpider(InitSpider): for row in csviter(response, self.delimiter, self.headers): ret = self.parse_row(response, row) - if isinstance(ret, (ScrapedItem, Request)): + if isinstance(ret, (BaseItem, Request)): ret = [ret] if not isinstance(ret, (list, tuple)): raise TypeError('You cannot return an "%s" object from a spider' % type(ret).__name__) diff --git a/scrapy/contrib/web/service.py b/scrapy/contrib/web/service.py index ac70c4cc0..69f3d70c1 100644 --- a/scrapy/contrib/web/service.py +++ b/scrapy/contrib/web/service.py @@ -6,7 +6,7 @@ from twisted.internet import defer from scrapy.core.engine import scrapyengine from scrapy.spider import spiders from scrapy.http import Request -from scrapy.item import ScrapedItem +from scrapy.item.models import BaseItem from scrapy.core.exceptions import NotConfigured from scrapy.conf import settings from scrapy.utils.misc import memoize, arg_to_iter @@ -63,7 +63,7 @@ def url_to_guid(httprequest): except Exception, ex: return _response(message='Error processing url') - guids = [i.guid for i in arg_to_iter(items) if isinstance(i, ScrapedItem)] + guids = [i.guid for i in arg_to_iter(items) if isinstance(i, BaseItem)] return _response(guids=guids) deferred = defer.Deferred().addCallbacks(_on_success, _on_error) diff --git a/scrapy/contrib_exp/newitem/models.py b/scrapy/contrib_exp/newitem/models.py index 9eca38210..73b92542b 100644 --- a/scrapy/contrib_exp/newitem/models.py +++ b/scrapy/contrib_exp/newitem/models.py @@ -1,6 +1,6 @@ from UserDict import DictMixin -from scrapy.item import ScrapedItem +from scrapy.item.models import BaseItem from scrapy.contrib_exp.newitem.fields import BaseField @@ -21,7 +21,7 @@ class _ItemMeta(type): return cls -class Item(DictMixin, ScrapedItem): +class Item(DictMixin, BaseItem): __metaclass__ = _ItemMeta diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7a872b79e..2be14fad9 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -9,7 +9,7 @@ from scrapy.utils.misc import arg_to_iter, load_object from scrapy.core.exceptions import IgnoreRequest, DropItem from scrapy.core import signals from scrapy.http import Request, Response -from scrapy.item import ScrapedItem +from scrapy.item.models import BaseItem from scrapy.spider.middleware import SpiderMiddlewareManager from scrapy import log from scrapy.stats import stats @@ -151,7 +151,7 @@ class Scraper(object): signals.send_catch_log(signal=signals.request_received, request=output, \ spider=spider) self.engine.crawl(request=output, spider=spider) - elif isinstance(output, ScrapedItem): + elif isinstance(output, BaseItem): log.msg("Scraped %s in <%s>" % (output, request.url), domain=domain) signals.send_catch_log(signal=signals.item_scraped, sender=self.__class__, \ item=output, spider=spider, response=response) @@ -162,7 +162,7 @@ class Scraper(object): elif output is None: pass else: - log.msg("Spider must return Request, ScrapedItem or None, got '%s' in %s" % \ + log.msg("Spider must return Request, BaseItem or None, got '%s' in %s" % \ (type(output).__name__, request), log.ERROR, domain=domain) def _check_propagated_failure(self, spider_failure, propagated_failure, request, spider): diff --git a/scrapy/item/__init__.py b/scrapy/item/__init__.py index 423ebfb01..35982e691 100644 --- a/scrapy/item/__init__.py +++ b/scrapy/item/__init__.py @@ -1 +1 @@ -from scrapy.item.models import ScrapedItem, ItemDelta +from scrapy.item.models import ScrapedItem diff --git a/scrapy/item/models.py b/scrapy/item/models.py index ed374f919..39cc0e7d7 100644 --- a/scrapy/item/models.py +++ b/scrapy/item/models.py @@ -1,9 +1,9 @@ -import copy +class BaseItem(object): + """Base class for all scraped items.""" + pass -class ScrapedItem(object): - """ - This is the base class for all scraped items. - """ + +class ScrapedItem(BaseItem): def __init__(self, data=None): """ @@ -24,12 +24,3 @@ class ScrapedItem(object): reprdict = dict(items for items in self.__dict__.iteritems() if not items[0].startswith('_')) return "%s(%s)" % (self.__class__.__name__, repr(reprdict)) - def __sub__(self, other): - raise NotImplementedError - - def copy(self): - """Create a new ScrapedItem based on the current one""" - return copy.deepcopy(self) - -class ItemDelta(object): - pass diff --git a/scrapy/item/pipeline.py b/scrapy/item/pipeline.py index ae18d979e..258998970 100644 --- a/scrapy/item/pipeline.py +++ b/scrapy/item/pipeline.py @@ -1,6 +1,6 @@ from scrapy import log from scrapy.core.exceptions import NotConfigured -from scrapy.item import ScrapedItem +from scrapy.item.models import BaseItem from scrapy.utils.misc import load_object from scrapy.utils.defer import defer_succeed, mustbe_deferred from scrapy.conf import settings @@ -39,8 +39,8 @@ class ItemPipelineManager(object): return defer_succeed(item) def next_stage(item, stages_left): - assert isinstance(item, ScrapedItem), \ - 'Item pipelines must return a ScrapedItem, got %s' % type(item).__name__ + assert isinstance(item, BaseItem), \ + 'Item pipelines must return a BaseItem, got %s' % type(item).__name__ if not stages_left: return item current_stage = stages_left.pop(0) diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 710e7d743..79fe79615 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -4,7 +4,7 @@ Helper functions for formatting and pretty printing some objects import sys import pprint as pypprint -from scrapy.item import ScrapedItem +from scrapy.item.models import BaseItem from scrapy.http import Request, Response nocolour = False @@ -35,7 +35,7 @@ def pformat(obj, *args, **kwargs): if isinstance(obj, (list, tuple)): return "".join([pformat(i, *args, **kwargs) for i in obj]) - elif isinstance(obj, (ScrapedItem, Request, Response)): + elif isinstance(obj, (BaseItem, Request, Response)): return pformat_dictobj(obj) else: return colorize(pypprint.pformat(obj))