Moved SetGUIDPipeline to contrib/item because it was unnecesary to put it on an exclusive module

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40350
This commit is contained in:
elpolilla 2008-10-29 01:35:23 +00:00
parent c73dc5ad6c
commit d61cd60756
3 changed files with 16 additions and 21 deletions

View File

@ -1 +1 @@
from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline
from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline, SetGUIDPipeline

View File

@ -5,8 +5,11 @@ useful in some Scrapy implementations
import hashlib
from pydispatch import dispatcher
from pprint import PrettyPrinter
from scrapy.item import ScrapedItem, ItemDelta
from scrapy.core import signals
from scrapy.core.exceptions import UsageError, DropItem
class ValidationError(DropItem):
@ -26,6 +29,18 @@ class ValidationPipeline(object):
item.validate()
return item
class SetGUIDPipeline(object):
def __init__(self):
self.spider = None
dispatcher.connect(self.domain_opened, signal=signals.domain_opened)
def domain_opened(self, domain, spider):
self.spider = spider
def process_item(self, domain, response, item):
self.spider.set_guid(item)
return item
class RobustScrapedItem(ScrapedItem):
"""
A more robust scraped item class with a built-in validation mechanism and

View File

@ -1,20 +0,0 @@
"""
This pipeline sets the guid for each scraped item.
It should be the previous to the validation pipeline (which is the last one).
"""
from pydispatch import dispatcher
from scrapy.core import signals
class SetGUIDPipeline(object):
def __init__(self):
self.spider = None
dispatcher.connect(self.domain_opened, signal=signals.domain_opened)
def domain_opened(self, domain, spider):
self.spider = spider
def process_item(self, domain, response, item):
self.spider.set_guid(item)
return item