From 27dd68a69093a2fae3d86d88631c3906fe070130 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Fri, 2 Sep 2011 13:06:59 -0300
Subject: [PATCH] added SpiderState extension

---
 docs/topics/settings.rst            |  1 +
 scrapy/contrib/spiderstate.py       | 33 +++++++++++++++++++++++++++++
 scrapy/settings/default_settings.py |  1 +
 3 files changed, 35 insertions(+)
 create mode 100644 scrapy/contrib/spiderstate.py

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index e19185a4c..f9d0b303c 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -599,6 +599,7 @@ Default::
         'scrapy.contrib.feedexport.FeedExporter': 0,
         'scrapy.contrib.spidercontext.SpiderContext': 0,
         'scrapy.contrib.logstats.LogStats': 0,
+        'scrapy.contrib.spiderstate.SpiderState': 0,
     }
 
 The list of available extensions. Keep in mind that some of them need to
diff --git a/scrapy/contrib/spiderstate.py b/scrapy/contrib/spiderstate.py
new file mode 100644
index 000000000..c351a6e59
--- /dev/null
+++ b/scrapy/contrib/spiderstate.py
@@ -0,0 +1,33 @@
+from __future__ import with_statement
+
+import os, cPickle as pickle
+
+from scrapy import signals
+from scrapy.exceptions import NotConfigured
+from scrapy.xlib.pydispatch import dispatcher
+
+class SpiderState(object):
+    """Stores and loads spider state"""
+
+    def __init__(self, jobdir):
+        self.statefn = os.path.join(jobdir, 'spider.state')
+        dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
+        dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        jobdir = crawler.settings.get('JOBDIR')
+        if not jobdir:
+            raise NotConfigured
+        return cls(jobdir)
+
+    def spider_closed(self, spider):
+        with open(self.statefn, 'wb') as f:
+            pickle.dump(spider.state, f, protocol=2)
+
+    def spider_opened(self, spider):
+        if os.path.exists(self.statefn):
+            with open(self.statefn) as f:
+                spider.state = pickle.load(f)
+        else:
+            spider.state = {}
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 415c7519d..2cb4ebcad 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -144,6 +144,7 @@ EXTENSIONS_BASE = {
     'scrapy.contrib.closespider.CloseSpider': 0,
     'scrapy.contrib.feedexport.FeedExporter': 0,
     'scrapy.contrib.logstats.LogStats': 0,
+    'scrapy.contrib.spiderstate.SpiderState': 0,
 }
 
 FEED_URI = None