From c70f050bdaeec6d2c32039c005511a32d3b21cc5 Mon Sep 17 00:00:00 2001 From: nramirezuy Date: Thu, 27 Jun 2013 11:45:29 -0300 Subject: [PATCH] from_crawler added to spiders --- scrapy/spidermanager.py | 6 +++++- scrapy/tests/test_spidermanager/__init__.py | 8 +++++--- .../tests/test_spidermanager/test_spiders/spider4.py | 10 ++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 scrapy/tests/test_spidermanager/test_spiders/spider4.py diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py index 6443c45fa..9ab570964 100644 --- a/scrapy/spidermanager.py +++ b/scrapy/spidermanager.py @@ -33,6 +33,7 @@ class SpiderManager(object): @classmethod def from_crawler(cls, crawler): sm = cls.from_settings(crawler.settings) + sm.crawler = crawler crawler.signals.connect(sm.close_spider, signals.spider_closed) return sm @@ -41,7 +42,10 @@ class SpiderManager(object): spcls = self._spiders[spider_name] except KeyError: raise KeyError("Spider not found: %s" % spider_name) - return spcls(**spider_kwargs) + if hasattr(self, 'crawler') and hasattr(spcls, 'from_crawler'): + return spcls.from_crawler(self.crawler, **spider_kwargs) + else: + return spcls(**spider_kwargs) def find_by_request(self, request): return [name for name, cls in self._spiders.iteritems() diff --git a/scrapy/tests/test_spidermanager/__init__.py b/scrapy/tests/test_spidermanager/__init__.py index 4b46022e6..c8697d3b5 100644 --- a/scrapy/tests/test_spidermanager/__init__.py +++ b/scrapy/tests/test_spidermanager/__init__.py @@ -1,6 +1,5 @@ import sys import os -import weakref import shutil from zope.interface.verify import verifyObject @@ -9,7 +8,6 @@ from twisted.trial import unittest # ugly hack to avoid cyclic imports of scrapy.spider when running this test # alone -import scrapy.spider from scrapy.interfaces import ISpiderManager from scrapy.spidermanager import SpiderManager from scrapy.http import Request @@ -36,7 +34,7 @@ class SpiderManagerTest(unittest.TestCase): def test_list(self): self.assertEqual(set(self.spiderman.list()), - set(['spider1', 'spider2', 'spider3'])) + set(['spider1', 'spider2', 'spider3', 'spider4'])) def test_create(self): spider1 = self.spiderman.create("spider1") @@ -66,3 +64,7 @@ class SpiderManagerTest(unittest.TestCase): def test_load_base_spider(self): self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider0']) assert len(self.spiderman._spiders) == 0 + + def test_load_from_crawler(self): + spider = self.spiderman.create('spider4', a='OK') + self.assertEqual(spider.a, 'OK') diff --git a/scrapy/tests/test_spidermanager/test_spiders/spider4.py b/scrapy/tests/test_spidermanager/test_spiders/spider4.py new file mode 100644 index 000000000..6f66ad37e --- /dev/null +++ b/scrapy/tests/test_spidermanager/test_spiders/spider4.py @@ -0,0 +1,10 @@ +from scrapy.spider import BaseSpider + +class Spider4(BaseSpider): + name = "spider4" + + @classmethod + def from_crawler(cls, crawler, **kwargs): + o = cls(**kwargs) + o.crawler = crawler + return o