From 5ce2b66c562b6ea511064e1579ee916530d4f920 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Tue, 1 Sep 2009 22:53:04 -0300
Subject: [PATCH] added proper recycling of spider resources to spider manager

---
 scrapy/contrib/spidermanager.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py
index c91735ca2..9b2a01dc3 100644
--- a/scrapy/contrib/spidermanager.py
+++ b/scrapy/contrib/spidermanager.py
@@ -7,7 +7,10 @@ import sys
 import urlparse
 
 from twisted.plugin import getCache
+from twisted.python.rebuild import rebuild
 
+from scrapy.xlib.pydispatch import dispatcher
+from scrapy.core import signals
 from scrapy.spider.models import ISpider
 from scrapy import log
 from scrapy.conf import settings
@@ -22,6 +25,7 @@ class TwistedPluginSpiderManager(object):
         self.default_domain = None
         self.force_domain = None
         self.spider_modules = None
+        dispatcher.connect(self._domain_closed, signal=signals.domain_closed)
 
     def fromdomain(self, domain_name):
         return self.asdict().get(domain_name)
@@ -107,3 +111,12 @@ class TwistedPluginSpiderManager(object):
         except KeyboardInterrupt:
             sys.stderr.write("Interrupted while loading Scrapy spiders\n")
             sys.exit(2)
+
+    def _domain_closed(self, domain, spider):
+        """Reload spider module to release any resources held on to by the
+        spider
+        """
+        module_name = spider.__module__
+        log.msg("reloading module %s" % module_name, domain=domain)
+        new_module = rebuild(sys.modules[module_name])
+        self._spiders[domain] = new_module.SPIDER