RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

spidermanager refactoring 

* Implements find/create method in Spider Manager API, removed fromdomain and fromurl

    This method is now in charge of spider resolution, it must return spider object
    from its argument or raise KeyError if no spider is found.

    This method obsoletes from_domain and from_url methods.

    The default implementation of resolve only searches against spider.name, it
    won't use spider.allowed_domains like the old fromdomain. This is the reason
    of why you must supply a spider if you want to crawl an url.

    Find methods returns only available spider names. Not spider instances.
    If no spider found returns empty list.

Affected modules:
    * command.models (force_domain)
        * removed spiders.force_domain
    * each command pass spider to crawl_* commands
    * command.commands.*
        * crawl
            * set spider from opts.spider if arg is url
            * group urls by spider to instance spider just once
        * genspider
            * use spiders.create() to check spider id
        * parse
            * log error if more than one spider found
    * core.manager
        * on crawl_* log message if multiple spiders found for url or request
    * shell
        * prints "Multiple found" if more than one spider found for url or request
        * populate_vars(): added spider keyword parameter

    * contrib.spidermanager:
        * removed fromdomain() & fromurl()
        * new create(spider_id) -> Spider. Raises KeyError if spider not found
        * new find_by_request(request) -> list(spiders)
This commit is contained in:
Rolando Espinoza La fuente 2010-04-01 17:16:38 -03:00
parent 8db67b17a3
commit dd477914db
7 changed files with 114 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
scrapy/command/commands/crawl.py
View File

@ -1,8 +1,12 @@
from scrapy import log
from scrapy.command import ScrapyCommand
from scrapy.core.manager import scrapymanager
from scrapy.conf import settings
from scrapy.http import Request
from scrapy.spider import spiders
from scrapy.utils.url import is_url
from collections import defaultdict
class Command(ScrapyCommand):
@ -25,12 +29,49 @@ class Command(ScrapyCommand):
settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
def run(self, args, opts):
if opts.spider:
spider = spiders.create(opts.spider)
else:
spider = None
# aggregate urls and domains
urls = []
domains = []
for arg in args:
# schedule arg as url or domain
if is_url(arg):
scrapymanager.crawl_url(arg)
urls.append(arg)
else:
scrapymanager.crawl_domain(arg)
domains.append(arg)
# schedule first domains
for dom in domains:
scrapymanager.crawl_domain(dom)
# if forced spider schedule urls directly
if spider:
for url in urls:
scrapymanager.crawl_url(url, spider)
else:
# group urls by spider
spider_urls = defaultdict(list)
find_by_url = lambda url: spiders.find_by_request(Request(url))
for url in urls:
spider_names = find_by_url(url)
if not spider_names:
log.msg('Could not find spider for url: %s' % url,
log.ERROR)
elif len(spider_names) > 1:
log.msg('More than one spider found for url: %s' % url,
log.ERROR)
else:
spider_urls[spider_names[0]].append(url)
# schedule grouped urls with same spider
for name, urls in spider_urls.iteritems():
# instance spider for each url-list
spider = spiders.create(name)
for url in urls:
scrapymanager.crawl_url(url, spider)
# crawl just scheduled arguments without keeping idle
scrapymanager.start()

16
scrapy/command/commands/genspider.py
View File

@ -59,11 +59,17 @@ class Command(ScrapyCommand):
module = sanitize_module_name(args[0])
domain = args[1]
spider = spiders.fromdomain(domain)
if spider and not opts.force:
print "Spider '%s' already exists in module:" % domain
print " %s" % spider.__module__
sys.exit(1)
# if spider already exists and not force option then halt
try:
spider = spiders.create(domain)
except KeyError:
pass
else:
if not opts.force:
print "Spider '%s' already exists in module:" % domain
print " %s" % spider.__module__
sys.exit(1)
template_file = self._find_template(opts.template)
if template_file:

13
scrapy/command/commands/parse.py
View File

@ -37,10 +37,17 @@ class Command(ScrapyCommand):
return item
def run_callback(self, spider, response, callback, args, opts):
spider = spiders.fromurl(response.url)
if not spider:
log.msg('Cannot find spider for url: %s' % response.url, level=log.ERROR)
spider_names = spiders.find_by_request(response.request)
if not spider_names:
log.msg('Cannot find spider for url: %s' % response.url,
level=log.ERROR)
return (), ()
elif len(spider_names) > 1:
log.msg('More than one spider found for url: %s' % response.url,
level=log.ERROR)
return (), ()
else:
spider = spiders.create(spider_names[0])
if callback:
callback_fcn = callback if callable(callback) else getattr(spider, callback, None)

4
scrapy/command/models.py
View File

@ -99,10 +99,6 @@ class ScrapyCommand(object):
if opts.nolog:
settings.overrides['LOG_ENABLED'] = False
if opts.spider:
from scrapy.spider import spiders
spiders.force_domain = opts.spider
if opts.pidfile:
with open(opts.pidfile, "w") as f:
f.write(str(os.getpid()))

35
scrapy/contrib/spidermanager.py
View File

@ -19,35 +19,34 @@ class TwistedPluginSpiderManager(object):
def __init__(self):
self.loaded = False
self.force_domain = None
self._invaliddict = {}
self._spiders = {}
def fromdomain(self, domain):
return self._spiders.get(domain)
def create(self, spider_id):
"""
Returns Spider instance by given identifier.
If not exists raises KeyError.
"""
#@@@ currently spider_id = domain
# if lookup fails let dict's KeyError exception propagate
return self._spiders[spider_id]
def fromurl(self, url):
if self.force_domain:
return self._spiders.get(self.force_domain)
domain = urlparse.urlparse(url).hostname
domain = str(domain).replace('www.', '')
if domain:
if domain in self._spiders: # try first locating by domain
return self._spiders[domain]
else: # else search spider by spider
plist = self._spiders.values()
for p in plist:
if url_is_from_spider(url, p):
return p
def find_by_request(self, request):
"""
Returns list of spiders ids that match given Request.
"""
# just find by request.url
return [domain for domain, spider in self._spiders.iteritems()
if url_is_from_spider(request.url, spider)]
def list(self):
"""Returns list of spiders available."""
return self._spiders.keys()
def load(self, spider_modules=None):
"""Load spiders from module directory."""
if spider_modules is None:
spider_modules = settings.getlist('SPIDER_MODULES')
self.spider_modules = spider_modules
self._invaliddict = {}
self._spiders = {}
modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules]

42
scrapy/core/manager.py
View File

@ -6,20 +6,13 @@ from scrapy.extension import extensions
from scrapy import log
from scrapy.http import Request
from scrapy.core.engine import scrapyengine
from scrapy.spider import BaseSpider, spiders
from scrapy.spider import spiders
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.url import is_url
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
class ExecutionManager(object):
"""Process a list of sites or urls.
This class should be used in a main for process a list of sites/urls.
It extracts products and could be used to store results in a database or
just for testing spiders.
"""
def __init__(self):
self.interrupted = False
self.configured = False
@ -45,29 +38,30 @@ class ExecutionManager(object):
def crawl_url(self, url, spider=None):
"""Schedule given url for crawling."""
spider = spider or spiders.fromurl(url)
if spider is None:
spider = self._create_spider_for_request(Request(url), log_none=True, \
log_multiple=True)
if spider:
requests = arg_to_iter(spider.make_requests_from_url(url))
self._crawl_requests(requests, spider)
else:
log.msg('Could not find spider for url: %s' % url, log.ERROR)
def crawl_request(self, request, spider=None):
"""Schedule request for crawling."""
assert self.configured, "Scrapy Manager not yet configured"
spider = spider or spiders.fromurl(request.url)
if spider is None:
spider = self._create_spider_for_request(request, log_none=True, \
log_multiple=True)
if spider:
scrapyengine.crawl(request, spider)
else:
log.msg('Could not find spider for request: %s' % url, log.ERROR)
def crawl_domain(self, domain):
"""Schedule given domain for crawling."""
spider = spiders.fromdomain(domain)
if spider:
self.crawl_spider(spider)
else:
try:
spider = spiders.create(domain)
except KeyError:
log.msg('Could not find spider for domain: %s' % domain, log.ERROR)
else:
self.crawl_spider(spider)
def crawl_spider(self, spider):
"""Schedule spider for crawling."""
@ -75,6 +69,7 @@ class ExecutionManager(object):
self._crawl_requests(requests, spider)
def _crawl_requests(self, requests, spider):
"""Shortcut to schedule a list of requests"""
for req in requests:
self.crawl_request(req, spider)
@ -90,6 +85,17 @@ class ExecutionManager(object):
self.interrupted = True
scrapyengine.stop()
def _create_spider_for_request(self, request, default=None, log_none=False, \
log_multiple=False):
spider_names = spiders.find_by_request(request)
if len(spider_names) == 1:
return spiders.create(spider_names[0])
if len(spider_names) > 1 and log_multiple:
log.msg('More than one spider found for: %s' % request, log.ERROR)
if len(spider_names) == 0 and log_none:
log.msg('Could not find spider for: %s' % request, log.ERROR)
return default
def _signal_shutdown(self, signum, _):
signame = signal_names[signum]
log.msg("Received %s, shutting down gracefully. Send again to force " \

12
scrapy/shell.py
View File

@ -35,6 +35,7 @@ def parse_url(url):
u = urlparse.urlparse(url)
return url
class Shell(object):
requires_project = False
@ -52,18 +53,21 @@ class Shell(object):
else:
url = parse_url(request_or_url)
request = Request(url)
spider = spiders.fromurl(url) or BaseSpider('default')
spider = scrapymanager._create_spider_for_request(request, \
BaseSpider('default'), log_multiple=True)
print "Fetching %s..." % request
response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, \
request, spider)
if response:
self.populate_vars(url, response, request)
self.populate_vars(url, response, request, spider)
if print_help:
self.print_help()
else:
print "Done - use shelp() to see available objects"
def populate_vars(self, url=None, response=None, request=None):
def populate_vars(self, url=None, response=None, request=None, spider=None):
item = self.item_class()
self.vars['item'] = item
if url:
@ -73,7 +77,7 @@ class Shell(object):
self.vars['url'] = url
self.vars['response'] = response
self.vars['request'] = request
self.vars['spider'] = spiders.fromurl(url)
self.vars['spider'] = spider
if not self.nofetch:
self.vars['fetch'] = self.fetch
self.vars['view'] = open_in_browser