added BaseItem as base item class, and moved extra functionality of ScrapedItem to RobustScrapedItem

scrapy/command/commands/parse.py

@@ -1,7 +1,7 @@
 from scrapy.command import ScrapyCommand
 from scrapy.fetcher import fetch
 from scrapy.http import Request
-from scrapy.item import ScrapedItem
+from scrapy.item.models import BaseItem
 from scrapy.spider import spiders
 from scrapy.utils import display
 from scrapy import log
@@ -42,7 +42,8 @@ class Command(ScrapyCommand):
 
             result = callback_fcn(response)
             links = [i for i in result if isinstance(i, Request)]
-            items = [self.pipeline_process(i, spider, opts) for i in result if isinstance(i, ScrapedItem)]
+            items = [self.pipeline_process(i, spider, opts) for i in result if \
+                     isinstance(i, BaseItem)]
             return items, links
 
         return (), ()

scrapy/contrib/item/models.py

@@ -3,11 +3,12 @@ This module contains some extra base models for scraped items which could be
 useful in some Scrapy implementations
 """
 
+import copy
 import hashlib
 
 from pprint import PrettyPrinter
 
-from scrapy.item import ScrapedItem, ItemDelta
+from scrapy.item import ScrapedItem
 from scrapy.item.adaptors import AdaptorPipe
 from scrapy.core.exceptions import DropItem
 from scrapy.utils.python import unique
@@ -194,6 +195,10 @@ class RobustScrapedItem(ScrapedItem):
         pipe.add_adaptor(adaptor, position)
         self.set_attrib_adaptors(attrib, pipe)
 
+    def copy(self):
+        """Create a new RobustScrapedItem based on the current one"""
+        return copy.deepcopy(self)
+
     def validate(self):
         """Method used to validate item attributes data"""
         raise NotImplemented
@@ -215,7 +220,7 @@ class RobustScrapedItem(ScrapedItem):
         return hash_.hexdigest()
 
 
-class RobustItemDelta(ItemDelta):
+class RobustItemDelta(object):
     """
     This class represents the difference between
     a pair of RobustScrapedItems.

scrapy/contrib/spiders/crawl.py

@@ -71,7 +71,7 @@ class CrawlSpider(InitSpider):
 
     def parse_start_url(self, response):
         """Overrideable callback function for processing start_urls. It must
-        return a list of ScrapedItems and/or Requests"""
+        return a list of BaseItem and/or Requests"""
         return []
 
     def process_results(self, response, results):

scrapy/contrib/spiders/feed.py

@@ -6,7 +6,7 @@ See documentation in docs/ref/spiders.rst
 """
 
 from scrapy.contrib.spiders.init import InitSpider
-from scrapy.item import ScrapedItem
+from scrapy.item.models import BaseItem
 from scrapy.http import Request
 from scrapy.utils.iterators import xmliter, csviter
 from scrapy.xpath.selector import XmlXPathSelector, HtmlXPathSelector
@@ -51,13 +51,13 @@ class XMLFeedSpider(InitSpider):
         """This method is called for the nodes matching the provided tag name
         (itertag). Receives the response and an XPathSelector for each node.
         Overriding this method is mandatory. Otherwise, you spider won't work.
-        This method must return either a ScrapedItem, a Request, or a list
+        This method must return either a BaseItem, a Request, or a list
         containing any of them.
         """
 
         for xSel in nodes:
             ret = self.parse_item(response, xSel)
-            if isinstance(ret, (ScrapedItem, Request)):
+            if isinstance(ret, (BaseItem, Request)):
                 ret = [ret]
             if not isinstance(ret, (list, tuple)):
                 raise TypeError('You cannot return an "%s" object from a spider' % type(ret).__name__)
@@ -121,7 +121,7 @@ class CSVFeedSpider(InitSpider):
 
         for row in csviter(response, self.delimiter, self.headers):
             ret = self.parse_row(response, row)
-            if isinstance(ret, (ScrapedItem, Request)):
+            if isinstance(ret, (BaseItem, Request)):
                 ret = [ret]
             if not isinstance(ret, (list, tuple)):
                 raise TypeError('You cannot return an "%s" object from a spider' % type(ret).__name__)

scrapy/contrib/web/service.py

@@ -6,7 +6,7 @@ from twisted.internet import defer
 from scrapy.core.engine import scrapyengine
 from scrapy.spider import spiders
 from scrapy.http import Request
-from scrapy.item import ScrapedItem
+from scrapy.item.models import BaseItem
 from scrapy.core.exceptions import NotConfigured
 from scrapy.conf import settings
 from scrapy.utils.misc import memoize, arg_to_iter
@@ -63,7 +63,7 @@ def url_to_guid(httprequest):
         except Exception, ex:
             return _response(message='Error processing url')
 
-        guids = [i.guid for i in arg_to_iter(items) if isinstance(i, ScrapedItem)]
+        guids = [i.guid for i in arg_to_iter(items) if isinstance(i, BaseItem)]
         return _response(guids=guids)
 
     deferred = defer.Deferred().addCallbacks(_on_success, _on_error)

scrapy/contrib_exp/newitem/models.py

@@ -1,6 +1,6 @@
 from UserDict import DictMixin
 
-from scrapy.item import ScrapedItem
+from scrapy.item.models import BaseItem
 from scrapy.contrib_exp.newitem.fields import BaseField
 
 
@@ -21,7 +21,7 @@ class _ItemMeta(type):
         return cls
 
 
-class Item(DictMixin, ScrapedItem):
+class Item(DictMixin, BaseItem):
 
     __metaclass__ = _ItemMeta
 

scrapy/core/scraper.py

@@ -9,7 +9,7 @@ from scrapy.utils.misc import arg_to_iter, load_object
 from scrapy.core.exceptions import IgnoreRequest, DropItem
 from scrapy.core import signals
 from scrapy.http import Request, Response
-from scrapy.item import ScrapedItem
+from scrapy.item.models import BaseItem
 from scrapy.spider.middleware import SpiderMiddlewareManager
 from scrapy import log
 from scrapy.stats import stats
@@ -151,7 +151,7 @@ class Scraper(object):
             signals.send_catch_log(signal=signals.request_received, request=output, \
                 spider=spider)
             self.engine.crawl(request=output, spider=spider)
-        elif isinstance(output, ScrapedItem):
+        elif isinstance(output, BaseItem):
             log.msg("Scraped %s in <%s>" % (output, request.url), domain=domain)
             signals.send_catch_log(signal=signals.item_scraped, sender=self.__class__, \
                 item=output, spider=spider, response=response)
@@ -162,7 +162,7 @@ class Scraper(object):
         elif output is None:
             pass
         else:
-            log.msg("Spider must return Request, ScrapedItem or None, got '%s' in %s" % \
+            log.msg("Spider must return Request, BaseItem or None, got '%s' in %s" % \
                 (type(output).__name__, request), log.ERROR, domain=domain)
 
     def _check_propagated_failure(self, spider_failure, propagated_failure, request, spider):

scrapy/item/__init__.py

@@ -1 +1 @@
-from scrapy.item.models import ScrapedItem, ItemDelta
+from scrapy.item.models import ScrapedItem

scrapy/item/models.py

@@ -1,9 +1,9 @@
-import copy
+class BaseItem(object):
+    """Base class for all scraped items."""
+    pass
 
-class ScrapedItem(object):
-    """
-    This is the base class for all scraped items.
-    """
+
+class ScrapedItem(BaseItem):
 
     def __init__(self, data=None):
         """
@@ -24,12 +24,3 @@ class ScrapedItem(object):
         reprdict = dict(items for items in self.__dict__.iteritems() if not items[0].startswith('_'))
         return "%s(%s)" % (self.__class__.__name__, repr(reprdict))
 
-    def __sub__(self, other):
-        raise NotImplementedError
-
-    def copy(self):
-        """Create a new ScrapedItem based on the current one"""
-        return copy.deepcopy(self)
-
-class ItemDelta(object):
-    pass

scrapy/item/pipeline.py

@@ -1,6 +1,6 @@
 from scrapy import log
 from scrapy.core.exceptions import NotConfigured
-from scrapy.item import ScrapedItem
+from scrapy.item.models import BaseItem
 from scrapy.utils.misc import load_object
 from scrapy.utils.defer import defer_succeed, mustbe_deferred
 from scrapy.conf import settings
@@ -39,8 +39,8 @@ class ItemPipelineManager(object):
             return defer_succeed(item)
 
         def next_stage(item, stages_left):
-            assert isinstance(item, ScrapedItem), \
-                'Item pipelines must return a ScrapedItem, got %s' % type(item).__name__
+            assert isinstance(item, BaseItem), \
+                'Item pipelines must return a BaseItem, got %s' % type(item).__name__
             if not stages_left:
                 return item
             current_stage = stages_left.pop(0)

scrapy/utils/display.py

@@ -4,7 +4,7 @@ Helper functions for formatting and pretty printing some objects
 import sys
 import pprint as pypprint
 
-from scrapy.item import ScrapedItem
+from scrapy.item.models import BaseItem
 from scrapy.http import Request, Response
 
 nocolour = False
@@ -35,7 +35,7 @@ def pformat(obj, *args, **kwargs):
 
     if isinstance(obj, (list, tuple)):
         return "".join([pformat(i, *args, **kwargs) for i in obj])
-    elif isinstance(obj, (ScrapedItem, Request, Response)):
+    elif isinstance(obj, (BaseItem, Request, Response)):
         return pformat_dictobj(obj)
     else:
         return colorize(pypprint.pformat(obj))