RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Replaced default spider manager (TwistedPluginSpiderManger) with a simpler one that doesn't depend on Twisted Plugins infrastructure.

This commit is contained in:
Ismael Carnales 2010-07-30 17:30:32 -03:00
parent 65c3de8e6a
commit e145ec686c
17 changed files with 34 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/intro/tutorial.rst
View File

@ -138,8 +138,6 @@ This is the code for our first Spider, save it in a file named
def parse(self, response):
filename = response.url.split("/")[-2]
open(filename, 'wb').write(response.body)
SPIDER = DmozSpider()
Crawling
--------
@ -370,8 +368,6 @@ Let's add this code to our spider::
link = site.select('a/@href').extract()
desc = site.select('text()').extract()
print title, link, desc
SPIDER = DmozSpider()
Now try crawling the dmoz.org domain again and you'll see sites being printed
in your output, run::
@ -418,8 +414,6 @@ scraped so far, the code for our Spider should be like this::
item['desc'] = site.select('text()').extract()
items.append(item)
return items
SPIDER = DmozSpider()
Now doing a crawl on the dmoz.org domain yields ``DmozItem``'s::

2
docs/topics/firebug.rst
View File

@ -93,8 +93,6 @@ This is how the spider would look so far::
# write the category page data extraction code here
pass
SPIDER = GoogleDirectorySpider()
Extracting the data
===================

11
docs/topics/spiders.rst
View File

@ -177,8 +177,6 @@ Let's see an example::
def parse(self, response):
self.log('A response from %s just arrived!' % response.url)
SPIDER = MySpider()
Another example returning multiples Requests and Items from a single callback::
from scrapy.selector import HtmlXPathSelector
@ -203,8 +201,6 @@ Another example returning multiples Requests and Items from a single callback::
for url in hxs.select('//a/@href').extract():
yield Request(url, callback=self.parse)
SPIDER = MySpider()
.. module:: scrapy.contrib.spiders
:synopsis: Collection of generic spiders
@ -292,8 +288,6 @@ Let's now take a look at an example CrawlSpider with rules::
item['description'] = hxs.select('//td[@id="item_description"]/text()').extract()
return item
SPIDER = MySpider()
This spider would start crawling example.com's home page, collecting category
links, and item links, parsing the latter with the ``parse_item`` method. For
@ -409,8 +403,6 @@ These spiders are pretty easy to use, let's have at one example::
item['description'] = node.select('description').extract()
return item
SPIDER = MySpider()
Basically what we did up there was creating a spider that downloads a feed from
the given ``start_urls``, and then iterates through each of its ``item`` tags,
prints them out, and stores some random data in an :class:`~scrapy.item.Item`.
@ -466,6 +458,3 @@ Let's see an example similar to the previous one, but using a
item['name'] = row['name']
item['description'] = row['description']
return item
SPIDER = MySpider()

3
examples/experimental/googledir/googledir/spiders/google_directory.py
View File

@ -36,6 +36,3 @@ class GoogleDirectorySpider(CrawlSpider):
# Here we populate the item and yield it
yield l.load_item()
SPIDER = GoogleDirectorySpider()

2
examples/experimental/imdb/imdb/spiders/imdb_site.py
View File

@ -137,5 +137,3 @@ class ImdbSiteSpider(CrawlSpider):
def _urljoin(self, response, url):
"""Helper to convert relative urls to absolute"""
return urljoin_rfc(response.url, url, response.encoding)
SPIDER = ImdbSiteSpider()

2
examples/googledir/googledir/spiders/google_directory.py
View File

@ -38,5 +38,3 @@ class GoogleDirectorySpider(CrawlSpider):
# Here we populate the item and yield it
yield l.load_item()
SPIDER = GoogleDirectorySpider()

2
scrapy/conf/default_settings.py
View File

@ -191,7 +191,7 @@ SCHEDULER_ORDER = 'DFO'
SERVICE_QUEUE = 'scrapy.core.queue.KeepAliveExecutionQueue'
SPIDER_MANAGER_CLASS = 'scrapy.contrib.spidermanager.TwistedPluginSpiderManager'
SPIDER_MANAGER_CLASS = 'scrapy.contrib.spidermanager.SpiderManager'
SPIDER_MIDDLEWARES = {}

64
scrapy/contrib/spidermanager.py
View File

@ -3,18 +3,16 @@ SpiderManager is the class which locates and manages all website-specific
spiders
"""
import sys
import inspect
from twisted.plugin import getCache
from twisted.python.rebuild import rebuild
from scrapy.spider.models import ISpider
from scrapy import log
from scrapy.conf import settings
from scrapy.utils.url import url_is_from_spider
from scrapy.utils.misc import walk_modules
from scrapy.spider import BaseSpider
class TwistedPluginSpiderManager(object):
"""Spider manager based in Twisted Plugin System"""
class SpiderManager(object):
def __init__(self):
self.loaded = False
@ -25,9 +23,7 @@ class TwistedPluginSpiderManager(object):
spider arguments. If the sipder name is not found, it raises a
KeyError.
"""
spider = self._spiders[spider_name]
spider.__dict__.update(spider_kwargs)
return spider
return self._spiders[spider_name](**spider_kwargs)
def find_by_request(self, request):
"""Returns list of spiders names that match the given Request"""
@ -60,47 +56,23 @@ class TwistedPluginSpiderManager(object):
return self._spiders.keys()
def load(self, spider_modules=None):
"""Load spiders from module directory."""
"""Load spiders from spider_modules or SPIDER_MODULES setting."""
if spider_modules is None:
spider_modules = settings.getlist('SPIDER_MODULES')
self.spider_modules = spider_modules
self._spiders = {}
modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules]
for module in modules:
for spider in self._getspiders(ISpider, module):
ISpider.validateInvariants(spider)
self._spiders[spider.name] = spider
self._spiders = {}
for name in self.spider_modules:
for module in walk_modules(name):
self._load_spiders(module)
self.loaded = True
def _getspiders(self, interface, package):
"""This is an override of twisted.plugin.getPlugin, because we're
interested in catching exceptions thrown when loading spiders such as
KeyboardInterrupt
"""
try:
allDropins = getCache(package)
for dropin in allDropins.itervalues():
for plugin in dropin.plugins:
adapted = interface(plugin, None)
if adapted is not None:
yield adapted
except KeyboardInterrupt:
sys.stderr.write("Interrupted while loading Scrapy spiders\n")
sys.exit(2)
def _load_spiders(self, module):
for obj in vars(module).itervalues():
if inspect.isclass(obj) and issubclass(obj, BaseSpider):
name = getattr(obj, 'name', None)
if name is not None:
self._spiders[name] = obj
def close_spider(self, spider):
"""Reload spider module to release any resources held on to by the
spider
"""
name = spider.name
if name not in self._spiders:
return
spider = self._spiders[name]
module_name = spider.__module__
module = sys.modules[module_name]
if hasattr(module, 'SPIDER'):
log.msg("Reloading module %s" % module_name, spider=spider, \
level=log.DEBUG)
new_module = rebuild(module, doLog=0)
self._spiders[name] = new_module.SPIDER
pass

8
scrapy/spider/models.py
View File

@ -6,25 +6,17 @@ See documentation in docs/topics/spiders.rst
import warnings
from zope.interface import Interface, Attribute, invariant, implements
from twisted.plugin import IPlugin
from scrapy import log
from scrapy.http import Request
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.trackref import object_ref
class ISpider(Interface, IPlugin) :
"""Interface used by TwistedPluginSpiderManager to discover spiders"""
pass
class BaseSpider(object_ref):
"""Base class for scrapy spiders. All spiders must inherit from this
class.
"""
implements(ISpider)
# XXX: class attributes kept for backwards compatibility
name = None
start_urls = []

3
scrapy/templates/spiders/basic.tmpl
View File

@ -9,6 +9,3 @@ class $classname(BaseSpider):
def parse(self, response):
pass
SPIDER = $classname()

2
scrapy/templates/spiders/crawl.tmpl
View File

@ -21,5 +21,3 @@ class $classname(CrawlSpider):
#i['name'] = hxs.select('//div[@id="name"]').extract()
#i['description'] = hxs.select('//div[@id="description"]').extract()
return i
SPIDER = $classname()

2
scrapy/templates/spiders/csvfeed.tmpl
View File

@ -18,5 +18,3 @@ class $classname(CSVFeedSpider):
#i['name'] = row['name']
#i['description'] = row['description']
return i
SPIDER = $classname()

2
scrapy/templates/spiders/xmlfeed.tmpl
View File

@ -12,5 +12,3 @@ class $classname(XMLFeedSpider):
#i['name'] = selector.select('name').extract()
#i['description'] = selector.select('description').extract()
return i
SPIDER = $classname()

36
scrapy/tests/test_contrib_spidermanager/__init__.py
View File

@ -7,13 +7,13 @@ from twisted.trial import unittest
# ugly hack to avoid cyclic imports of scrapy.spider when running this test
# alone
import scrapy.spider
from scrapy.contrib.spidermanager import TwistedPluginSpiderManager
import scrapy.spider
from scrapy.contrib.spidermanager import SpiderManager
from scrapy.http import Request
module_dir = os.path.dirname(os.path.abspath(__file__))
class TwistedPluginSpiderManagerTest(unittest.TestCase):
class SpiderManagerTest(unittest.TestCase):
def setUp(self):
orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
@ -22,7 +22,7 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase):
self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
shutil.copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(self.tmpdir)
self.spiderman = TwistedPluginSpiderManager()
self.spiderman = SpiderManager()
assert not self.spiderman.loaded
self.spiderman.load(['test_spiders_xxx'])
assert self.spiderman.loaded
@ -32,7 +32,7 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase):
sys.path.remove(self.tmpdir)
def test_list(self):
self.assertEqual(set(self.spiderman.list()),
self.assertEqual(set(self.spiderman.list()),
set(['spider1', 'spider2']))
def test_create(self):
@ -42,14 +42,6 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase):
self.assertEqual(spider2.__class__.__name__, 'Spider2')
self.assertEqual(spider2.foo, 'bar')
def test_create_uses_cache(self):
# TwistedPluginSpiderManager uses an internal cache which is
# invalidated in close_spider() but this isn't necessarily the best
# thing to do in all cases.
spider1 = self.spiderman.create("spider1")
spider2 = self.spiderman.create("spider1")
assert spider1 is spider2
def test_find_by_request(self):
self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')),
['spider1'])
@ -60,16 +52,10 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase):
self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy999.org/test')),
[])
def test_close_spider_remove_refs(self):
spider = self.spiderman.create("spider1")
wref = weakref.ref(spider)
assert wref()
self.spiderman.close_spider(spider)
del spider
assert not wref()
def test_load_spider_module(self):
self.spiderman.load(['scrapy.tests.test_contrib_spidermanager.test_spiders.spider1'])
assert len(self.spiderman._spiders) == 1
def test_close_spider_invalidates_cache(self):
spider1 = self.spiderman.create("spider1")
self.spiderman.close_spider(spider1)
spider2 = self.spiderman.create("spider1")
assert spider1 is not spider2
def test_load_base_spider(self):
self.spiderman.load(['scrapy.tests.test_contrib_spidermanager.test_spiders.spider0'])
assert len(self.spiderman._spiders) == 0

4
scrapy/tests/test_contrib_spidermanager/test_spiders/spider0.py Normal file
View File

@ -0,0 +1,4 @@
from scrapy.spider import BaseSpider
class Spider0(BaseSpider):
allowed_domains = ["scrapy1.org", "scrapy3.org"]

2
scrapy/tests/test_contrib_spidermanager/test_spiders/spider1.py
View File

@ -3,5 +3,3 @@ from scrapy.spider import BaseSpider
class Spider1(BaseSpider):
name = "spider1"
allowed_domains = ["scrapy1.org", "scrapy3.org"]
SPIDER = Spider1()

2
scrapy/tests/test_contrib_spidermanager/test_spiders/spider2.py
View File

@ -3,5 +3,3 @@ from scrapy.spider import BaseSpider
class Spider2(BaseSpider):
name = "spider2"
allowed_domains = ["scrapy2.org", "scrapy3.org"]
SPIDER = Spider2()