From d61cd6075631a55f6351196944ae5ad7af2baf5c Mon Sep 17 00:00:00 2001 From: elpolilla Date: Wed, 29 Oct 2008 01:35:23 +0000 Subject: [PATCH] Moved SetGUIDPipeline to contrib/item because it was unnecesary to put it on an exclusive module --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40350 --- scrapy/trunk/scrapy/contrib/item/__init__.py | 2 +- scrapy/trunk/scrapy/contrib/item/models.py | 15 ++++++++++++++ .../trunk/scrapy/contrib/pipeline/setguid.py | 20 ------------------- 3 files changed, 16 insertions(+), 21 deletions(-) delete mode 100644 scrapy/trunk/scrapy/contrib/pipeline/setguid.py diff --git a/scrapy/trunk/scrapy/contrib/item/__init__.py b/scrapy/trunk/scrapy/contrib/item/__init__.py index a6226fec9..8f029a687 100644 --- a/scrapy/trunk/scrapy/contrib/item/__init__.py +++ b/scrapy/trunk/scrapy/contrib/item/__init__.py @@ -1 +1 @@ -from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline +from scrapy.contrib.item.models import RobustScrapedItem, RobustItemDelta, ValidationError, ValidationPipeline, SetGUIDPipeline diff --git a/scrapy/trunk/scrapy/contrib/item/models.py b/scrapy/trunk/scrapy/contrib/item/models.py index 2530d6814..378ca208a 100644 --- a/scrapy/trunk/scrapy/contrib/item/models.py +++ b/scrapy/trunk/scrapy/contrib/item/models.py @@ -5,8 +5,11 @@ useful in some Scrapy implementations import hashlib +from pydispatch import dispatcher from pprint import PrettyPrinter + from scrapy.item import ScrapedItem, ItemDelta +from scrapy.core import signals from scrapy.core.exceptions import UsageError, DropItem class ValidationError(DropItem): @@ -26,6 +29,18 @@ class ValidationPipeline(object): item.validate() return item +class SetGUIDPipeline(object): + def __init__(self): + self.spider = None + dispatcher.connect(self.domain_opened, signal=signals.domain_opened) + + def domain_opened(self, domain, spider): + self.spider = spider + + def process_item(self, domain, response, item): + self.spider.set_guid(item) + return item + class RobustScrapedItem(ScrapedItem): """ A more robust scraped item class with a built-in validation mechanism and diff --git a/scrapy/trunk/scrapy/contrib/pipeline/setguid.py b/scrapy/trunk/scrapy/contrib/pipeline/setguid.py deleted file mode 100644 index 275ce8aca..000000000 --- a/scrapy/trunk/scrapy/contrib/pipeline/setguid.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -This pipeline sets the guid for each scraped item. -It should be the previous to the validation pipeline (which is the last one). -""" - -from pydispatch import dispatcher -from scrapy.core import signals - -class SetGUIDPipeline(object): - def __init__(self): - self.spider = None - dispatcher.connect(self.domain_opened, signal=signals.domain_opened) - - def domain_opened(self, domain, spider): - self.spider = spider - - def process_item(self, domain, response, item): - self.spider.set_guid(item) - return item -