mirror of https://github.com/scrapy/scrapy.git
Moved SetGUIDPipeline to contrib/item because it was unnecesary to put it on an exclusive module
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40350
This commit is contained in:
parent
c73dc5ad6c
commit
d61cd60756
|
|
@ -1 +1 @@
|
|||
from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline
|
||||
from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline, SetGUIDPipeline
|
||||
|
|
|
|||
|
|
@ -5,8 +5,11 @@ useful in some Scrapy implementations
|
|||
|
||||
import hashlib
|
||||
|
||||
from pydispatch import dispatcher
|
||||
from pprint import PrettyPrinter
|
||||
|
||||
from scrapy.item import ScrapedItem, ItemDelta
|
||||
from scrapy.core import signals
|
||||
from scrapy.core.exceptions import UsageError, DropItem
|
||||
|
||||
class ValidationError(DropItem):
|
||||
|
|
@ -26,6 +29,18 @@ class ValidationPipeline(object):
|
|||
item.validate()
|
||||
return item
|
||||
|
||||
class SetGUIDPipeline(object):
|
||||
def __init__(self):
|
||||
self.spider = None
|
||||
dispatcher.connect(self.domain_opened, signal=signals.domain_opened)
|
||||
|
||||
def domain_opened(self, domain, spider):
|
||||
self.spider = spider
|
||||
|
||||
def process_item(self, domain, response, item):
|
||||
self.spider.set_guid(item)
|
||||
return item
|
||||
|
||||
class RobustScrapedItem(ScrapedItem):
|
||||
"""
|
||||
A more robust scraped item class with a built-in validation mechanism and
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
"""
|
||||
This pipeline sets the guid for each scraped item.
|
||||
It should be the previous to the validation pipeline (which is the last one).
|
||||
"""
|
||||
|
||||
from pydispatch import dispatcher
|
||||
from scrapy.core import signals
|
||||
|
||||
class SetGUIDPipeline(object):
|
||||
def __init__(self):
|
||||
self.spider = None
|
||||
dispatcher.connect(self.domain_opened, signal=signals.domain_opened)
|
||||
|
||||
def domain_opened(self, domain, spider):
|
||||
self.spider = spider
|
||||
|
||||
def process_item(self, domain, response, item):
|
||||
self.spider.set_guid(item)
|
||||
return item
|
||||
|
||||
Loading…
Reference in New Issue