From b3050ca0e4071649f0d8d522bfef537528a4bd25 Mon Sep 17 00:00:00 2001
From: Daniel Grana <dangra@gmail.com>
Date: Thu, 29 Jan 2009 19:18:03 +0000
Subject: [PATCH] pipeline: remove open_domain/close_domain hooks, use
 domain_open and domain_closed signals instead. docs updated.

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40798
---
 scrapy/trunk/docs/topics/item-pipeline.rst    | 49 +++++++++++++------
 scrapy/trunk/scrapy/contrib/pipeline/media.py |  8 ++-
 scrapy/trunk/scrapy/item/pipeline.py          |  6 ---
 3 files changed, 40 insertions(+), 23 deletions(-)

diff --git a/scrapy/trunk/docs/topics/item-pipeline.rst b/scrapy/trunk/docs/topics/item-pipeline.rst
index 9b4c73084..033e98e3f 100644
--- a/scrapy/trunk/docs/topics/item-pipeline.rst
+++ b/scrapy/trunk/docs/topics/item-pipeline.rst
@@ -33,21 +33,6 @@ exception. Dropped items are no longer processed by further pipeline
 components.
 
 
-The class can also define the followin optional methods:
-
-.. method:: open_domain(domain)
-
-``domain`` is a string which identifies the spider opened
-
-This method is called when a spider is opened for crawling.
-
-.. method:: close_domain(domain)
-
-``domain`` is a string which identifies the spider closed
-
-This method is called when a spider is closed.
-
-
 Item pipeline example
 =====================
 
@@ -69,3 +54,37 @@ attribute), and drops those items which don't contain a price::
             else:
                 raise DropItem("Missing price in %s" % item)
 
+
+Item pipeline example with resources per domain
+===============================================
+
+Sometimes you need to keep resources about the items processed grouped per
+domain, and delete those resource when a domain finish.
+
+An example is a filter that looks for duplicate items, and drops those items
+that were already processed. Let say that our items has an unique id, but our
+spider returns multiples items with the same id::
+
+
+    from pydispatch import dispatcher
+    from scrapy.core import signals
+    from scrapy.core.exceptions import DropItem
+
+    class DuplicatesPipeline(object):
+        def __init__(self):
+            self.domaininfo = {}
+            dispatcher.connect(self.domain_open, signals.domain_open)
+            dispatcher.connect(self.domain_closed, signals.domain_closed)
+
+        def domain_open(self, domain):
+            self.duplicates[domain] = set()
+
+        def domain_closed(self, domain):
+            del self.duplicates[domain]
+
+        def process_item(self, domain, item):
+            if item.id in self.duplicates[domain]:
+                raise DropItem("Duplicate item found: %s" % item)
+            else:
+                self.duplicates[domain].add(item.id)
+                return item
diff --git a/scrapy/trunk/scrapy/contrib/pipeline/media.py b/scrapy/trunk/scrapy/contrib/pipeline/media.py
index 0cb9a3412..2e1a025a7 100644
--- a/scrapy/trunk/scrapy/contrib/pipeline/media.py
+++ b/scrapy/trunk/scrapy/contrib/pipeline/media.py
@@ -1,7 +1,9 @@
+from pydispatch import dispatcher
 from twisted.internet import defer
 
 from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy import log
+from scrapy.core import signals
 from scrapy.core.engine import scrapyengine
 from scrapy.utils.request import request_fingerprint
 from scrapy.spider import spiders
@@ -19,11 +21,13 @@ class MediaPipeline(object):
 
     def __init__(self):
         self.domaininfo = {}
+        dispatcher.connect(self.domain_open, signals.domain_open)
+        dispatcher.connect(self.domain_closed, signals.domain_closed)
 
-    def open_domain(self, domain):
+    def domain_open(self, domain):
         self.domaininfo[domain] = self.DomainInfo(domain)
 
-    def close_domain(self, domain):
+    def domain_closed(self, domain):
         del self.domaininfo[domain]
 
     def process_item(self, domain, item):
diff --git a/scrapy/trunk/scrapy/item/pipeline.py b/scrapy/trunk/scrapy/item/pipeline.py
index 787bc61bc..fd86fee4b 100644
--- a/scrapy/trunk/scrapy/item/pipeline.py
+++ b/scrapy/trunk/scrapy/item/pipeline.py
@@ -30,14 +30,8 @@ class ItemPipelineManager(object):
 
     def open_domain(self, domain):
         self.domaininfo[domain] = set()
-        for pipe in self.pipeline:
-            if hasattr(pipe, 'open_domain'):
-                pipe.open_domain(domain)
 
     def close_domain(self, domain):
-        for pipe in self.pipeline:
-            if hasattr(pipe, 'close_domain'):
-                pipe.close_domain(domain)
         del self.domaininfo[domain]
 
     def is_idle(self):