diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index dd79756f1..bd091c445 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -138,8 +138,6 @@ This is the code for our first Spider, save it in a file named def parse(self, response): filename = response.url.split("/")[-2] open(filename, 'wb').write(response.body) - - SPIDER = DmozSpider() Crawling -------- @@ -370,8 +368,6 @@ Let's add this code to our spider:: link = site.select('a/@href').extract() desc = site.select('text()').extract() print title, link, desc - - SPIDER = DmozSpider() Now try crawling the dmoz.org domain again and you'll see sites being printed in your output, run:: @@ -418,8 +414,6 @@ scraped so far, the code for our Spider should be like this:: item['desc'] = site.select('text()').extract() items.append(item) return items - - SPIDER = DmozSpider() Now doing a crawl on the dmoz.org domain yields ``DmozItem``'s:: diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 649b2667e..065304280 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -93,8 +93,6 @@ This is how the spider would look so far:: # write the category page data extraction code here pass - SPIDER = GoogleDirectorySpider() - Extracting the data =================== diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index a79f1795e..fafd76788 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -177,8 +177,6 @@ Let's see an example:: def parse(self, response): self.log('A response from %s just arrived!' % response.url) - SPIDER = MySpider() - Another example returning multiples Requests and Items from a single callback:: from scrapy.selector import HtmlXPathSelector @@ -203,8 +201,6 @@ Another example returning multiples Requests and Items from a single callback:: for url in hxs.select('//a/@href').extract(): yield Request(url, callback=self.parse) - SPIDER = MySpider() - .. module:: scrapy.contrib.spiders :synopsis: Collection of generic spiders @@ -292,8 +288,6 @@ Let's now take a look at an example CrawlSpider with rules:: item['description'] = hxs.select('//td[@id="item_description"]/text()').extract() return item - SPIDER = MySpider() - This spider would start crawling example.com's home page, collecting category links, and item links, parsing the latter with the ``parse_item`` method. For @@ -409,8 +403,6 @@ These spiders are pretty easy to use, let's have at one example:: item['description'] = node.select('description').extract() return item - SPIDER = MySpider() - Basically what we did up there was creating a spider that downloads a feed from the given ``start_urls``, and then iterates through each of its ``item`` tags, prints them out, and stores some random data in an :class:`~scrapy.item.Item`. @@ -466,6 +458,3 @@ Let's see an example similar to the previous one, but using a item['name'] = row['name'] item['description'] = row['description'] return item - - SPIDER = MySpider() - diff --git a/examples/experimental/googledir/googledir/spiders/google_directory.py b/examples/experimental/googledir/googledir/spiders/google_directory.py index 2ed7c52a6..53abcbd21 100644 --- a/examples/experimental/googledir/googledir/spiders/google_directory.py +++ b/examples/experimental/googledir/googledir/spiders/google_directory.py @@ -36,6 +36,3 @@ class GoogleDirectorySpider(CrawlSpider): # Here we populate the item and yield it yield l.load_item() - -SPIDER = GoogleDirectorySpider() - diff --git a/examples/experimental/imdb/imdb/spiders/imdb_site.py b/examples/experimental/imdb/imdb/spiders/imdb_site.py index 8c2ebcd01..fed14ab07 100644 --- a/examples/experimental/imdb/imdb/spiders/imdb_site.py +++ b/examples/experimental/imdb/imdb/spiders/imdb_site.py @@ -137,5 +137,3 @@ class ImdbSiteSpider(CrawlSpider): def _urljoin(self, response, url): """Helper to convert relative urls to absolute""" return urljoin_rfc(response.url, url, response.encoding) - -SPIDER = ImdbSiteSpider() diff --git a/examples/googledir/googledir/spiders/google_directory.py b/examples/googledir/googledir/spiders/google_directory.py index b214be283..ceb370e3e 100644 --- a/examples/googledir/googledir/spiders/google_directory.py +++ b/examples/googledir/googledir/spiders/google_directory.py @@ -38,5 +38,3 @@ class GoogleDirectorySpider(CrawlSpider): # Here we populate the item and yield it yield l.load_item() - -SPIDER = GoogleDirectorySpider() diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 4a666920c..7f018603b 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -191,7 +191,7 @@ SCHEDULER_ORDER = 'DFO' SERVICE_QUEUE = 'scrapy.core.queue.KeepAliveExecutionQueue' -SPIDER_MANAGER_CLASS = 'scrapy.contrib.spidermanager.TwistedPluginSpiderManager' +SPIDER_MANAGER_CLASS = 'scrapy.contrib.spidermanager.SpiderManager' SPIDER_MIDDLEWARES = {} diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py index 263f40f36..7b54c8173 100644 --- a/scrapy/contrib/spidermanager.py +++ b/scrapy/contrib/spidermanager.py @@ -3,18 +3,16 @@ SpiderManager is the class which locates and manages all website-specific spiders """ -import sys +import inspect -from twisted.plugin import getCache -from twisted.python.rebuild import rebuild - -from scrapy.spider.models import ISpider from scrapy import log from scrapy.conf import settings from scrapy.utils.url import url_is_from_spider +from scrapy.utils.misc import walk_modules +from scrapy.spider import BaseSpider -class TwistedPluginSpiderManager(object): - """Spider manager based in Twisted Plugin System""" + +class SpiderManager(object): def __init__(self): self.loaded = False @@ -25,9 +23,7 @@ class TwistedPluginSpiderManager(object): spider arguments. If the sipder name is not found, it raises a KeyError. """ - spider = self._spiders[spider_name] - spider.__dict__.update(spider_kwargs) - return spider + return self._spiders[spider_name](**spider_kwargs) def find_by_request(self, request): """Returns list of spiders names that match the given Request""" @@ -60,47 +56,23 @@ class TwistedPluginSpiderManager(object): return self._spiders.keys() def load(self, spider_modules=None): - """Load spiders from module directory.""" + """Load spiders from spider_modules or SPIDER_MODULES setting.""" if spider_modules is None: spider_modules = settings.getlist('SPIDER_MODULES') self.spider_modules = spider_modules - self._spiders = {} - modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] - for module in modules: - for spider in self._getspiders(ISpider, module): - ISpider.validateInvariants(spider) - self._spiders[spider.name] = spider + self._spiders = {} + for name in self.spider_modules: + for module in walk_modules(name): + self._load_spiders(module) self.loaded = True - def _getspiders(self, interface, package): - """This is an override of twisted.plugin.getPlugin, because we're - interested in catching exceptions thrown when loading spiders such as - KeyboardInterrupt - """ - try: - allDropins = getCache(package) - for dropin in allDropins.itervalues(): - for plugin in dropin.plugins: - adapted = interface(plugin, None) - if adapted is not None: - yield adapted - except KeyboardInterrupt: - sys.stderr.write("Interrupted while loading Scrapy spiders\n") - sys.exit(2) + def _load_spiders(self, module): + for obj in vars(module).itervalues(): + if inspect.isclass(obj) and issubclass(obj, BaseSpider): + name = getattr(obj, 'name', None) + if name is not None: + self._spiders[name] = obj def close_spider(self, spider): - """Reload spider module to release any resources held on to by the - spider - """ - name = spider.name - if name not in self._spiders: - return - spider = self._spiders[name] - module_name = spider.__module__ - module = sys.modules[module_name] - if hasattr(module, 'SPIDER'): - log.msg("Reloading module %s" % module_name, spider=spider, \ - level=log.DEBUG) - new_module = rebuild(module, doLog=0) - self._spiders[name] = new_module.SPIDER + pass diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index d0be6f901..a58f0a0c9 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -6,25 +6,17 @@ See documentation in docs/topics/spiders.rst import warnings -from zope.interface import Interface, Attribute, invariant, implements -from twisted.plugin import IPlugin - from scrapy import log from scrapy.http import Request from scrapy.utils.misc import arg_to_iter from scrapy.utils.trackref import object_ref -class ISpider(Interface, IPlugin) : - """Interface used by TwistedPluginSpiderManager to discover spiders""" - pass class BaseSpider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this class. """ - implements(ISpider) - # XXX: class attributes kept for backwards compatibility name = None start_urls = [] diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 246015466..63ddce401 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -9,6 +9,3 @@ class $classname(BaseSpider): def parse(self, response): pass - -SPIDER = $classname() - diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 578779c06..d0036443f 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -21,5 +21,3 @@ class $classname(CrawlSpider): #i['name'] = hxs.select('//div[@id="name"]').extract() #i['description'] = hxs.select('//div[@id="description"]').extract() return i - -SPIDER = $classname() diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index c9a723000..7551686a6 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -18,5 +18,3 @@ class $classname(CSVFeedSpider): #i['name'] = row['name'] #i['description'] = row['description'] return i - -SPIDER = $classname() diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index f5ecbd707..5e16d2325 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -12,5 +12,3 @@ class $classname(XMLFeedSpider): #i['name'] = selector.select('name').extract() #i['description'] = selector.select('description').extract() return i - -SPIDER = $classname() diff --git a/scrapy/tests/test_contrib_spidermanager/__init__.py b/scrapy/tests/test_contrib_spidermanager/__init__.py index 3f4cff83a..930ceed54 100644 --- a/scrapy/tests/test_contrib_spidermanager/__init__.py +++ b/scrapy/tests/test_contrib_spidermanager/__init__.py @@ -7,13 +7,13 @@ from twisted.trial import unittest # ugly hack to avoid cyclic imports of scrapy.spider when running this test # alone -import scrapy.spider -from scrapy.contrib.spidermanager import TwistedPluginSpiderManager +import scrapy.spider +from scrapy.contrib.spidermanager import SpiderManager from scrapy.http import Request module_dir = os.path.dirname(os.path.abspath(__file__)) -class TwistedPluginSpiderManagerTest(unittest.TestCase): +class SpiderManagerTest(unittest.TestCase): def setUp(self): orig_spiders_dir = os.path.join(module_dir, 'test_spiders') @@ -22,7 +22,7 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase): self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') shutil.copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) - self.spiderman = TwistedPluginSpiderManager() + self.spiderman = SpiderManager() assert not self.spiderman.loaded self.spiderman.load(['test_spiders_xxx']) assert self.spiderman.loaded @@ -32,7 +32,7 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase): sys.path.remove(self.tmpdir) def test_list(self): - self.assertEqual(set(self.spiderman.list()), + self.assertEqual(set(self.spiderman.list()), set(['spider1', 'spider2'])) def test_create(self): @@ -42,14 +42,6 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase): self.assertEqual(spider2.__class__.__name__, 'Spider2') self.assertEqual(spider2.foo, 'bar') - def test_create_uses_cache(self): - # TwistedPluginSpiderManager uses an internal cache which is - # invalidated in close_spider() but this isn't necessarily the best - # thing to do in all cases. - spider1 = self.spiderman.create("spider1") - spider2 = self.spiderman.create("spider1") - assert spider1 is spider2 - def test_find_by_request(self): self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')), ['spider1']) @@ -60,16 +52,10 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase): self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy999.org/test')), []) - def test_close_spider_remove_refs(self): - spider = self.spiderman.create("spider1") - wref = weakref.ref(spider) - assert wref() - self.spiderman.close_spider(spider) - del spider - assert not wref() + def test_load_spider_module(self): + self.spiderman.load(['scrapy.tests.test_contrib_spidermanager.test_spiders.spider1']) + assert len(self.spiderman._spiders) == 1 - def test_close_spider_invalidates_cache(self): - spider1 = self.spiderman.create("spider1") - self.spiderman.close_spider(spider1) - spider2 = self.spiderman.create("spider1") - assert spider1 is not spider2 + def test_load_base_spider(self): + self.spiderman.load(['scrapy.tests.test_contrib_spidermanager.test_spiders.spider0']) + assert len(self.spiderman._spiders) == 0 diff --git a/scrapy/tests/test_contrib_spidermanager/test_spiders/spider0.py b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider0.py new file mode 100644 index 000000000..442d690d1 --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider0.py @@ -0,0 +1,4 @@ +from scrapy.spider import BaseSpider + +class Spider0(BaseSpider): + allowed_domains = ["scrapy1.org", "scrapy3.org"] diff --git a/scrapy/tests/test_contrib_spidermanager/test_spiders/spider1.py b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider1.py index 0a9b60989..f38af6ba6 100644 --- a/scrapy/tests/test_contrib_spidermanager/test_spiders/spider1.py +++ b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider1.py @@ -3,5 +3,3 @@ from scrapy.spider import BaseSpider class Spider1(BaseSpider): name = "spider1" allowed_domains = ["scrapy1.org", "scrapy3.org"] - -SPIDER = Spider1() diff --git a/scrapy/tests/test_contrib_spidermanager/test_spiders/spider2.py b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider2.py index 52023277f..7af1441e7 100644 --- a/scrapy/tests/test_contrib_spidermanager/test_spiders/spider2.py +++ b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider2.py @@ -3,5 +3,3 @@ from scrapy.spider import BaseSpider class Spider2(BaseSpider): name = "spider2" allowed_domains = ["scrapy2.org", "scrapy3.org"] - -SPIDER = Spider2()