mirror of https://github.com/scrapy/scrapy.git
115 lines
4.1 KiB
Python
115 lines
4.1 KiB
Python
import signal
|
|
|
|
from twisted.internet import reactor
|
|
|
|
from scrapy.extension import extensions
|
|
from scrapy import log
|
|
from scrapy.http import Request
|
|
from scrapy.core.engine import scrapyengine
|
|
from scrapy.spider import spiders
|
|
from scrapy.utils.misc import arg_to_iter
|
|
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
|
|
|
|
|
|
class ExecutionManager(object):
|
|
|
|
def __init__(self):
|
|
self.interrupted = False
|
|
self.configured = False
|
|
self.control_reactor = True
|
|
|
|
def configure(self, control_reactor=True):
|
|
self.control_reactor = control_reactor
|
|
if control_reactor:
|
|
install_shutdown_handlers(self._signal_shutdown)
|
|
reactor.addSystemEventTrigger('before', 'shutdown', scrapyengine.stop)
|
|
|
|
if not log.started:
|
|
log.start()
|
|
if not extensions.loaded:
|
|
extensions.load()
|
|
if not spiders.loaded:
|
|
spiders.load()
|
|
log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys()),
|
|
level=log.DEBUG)
|
|
|
|
scrapyengine.configure()
|
|
self.configured = True
|
|
|
|
def crawl_url(self, url, spider=None):
|
|
"""Schedule given url for crawling."""
|
|
if spider is None:
|
|
spider = self._create_spider_for_request(Request(url), log_none=True, \
|
|
log_multiple=True)
|
|
if spider:
|
|
requests = arg_to_iter(spider.make_requests_from_url(url))
|
|
self._crawl_requests(requests, spider)
|
|
|
|
def crawl_request(self, request, spider=None):
|
|
"""Schedule request for crawling."""
|
|
assert self.configured, "Scrapy Manager not yet configured"
|
|
if spider is None:
|
|
spider = self._create_spider_for_request(request, log_none=True, \
|
|
log_multiple=True)
|
|
if spider:
|
|
scrapyengine.crawl(request, spider)
|
|
|
|
def crawl_spider_name(self, name):
|
|
"""Schedule given spider by name for crawling."""
|
|
try:
|
|
spider = spiders.create(name)
|
|
except KeyError:
|
|
log.msg('Could not find spider: %s' % name, log.ERROR)
|
|
else:
|
|
self.crawl_spider(spider)
|
|
|
|
def crawl_spider(self, spider):
|
|
"""Schedule spider for crawling."""
|
|
requests = spider.start_requests()
|
|
self._crawl_requests(requests, spider)
|
|
|
|
def _crawl_requests(self, requests, spider):
|
|
"""Shortcut to schedule a list of requests"""
|
|
for req in requests:
|
|
self.crawl_request(req, spider)
|
|
|
|
def start(self, keep_alive=False):
|
|
"""Start the scrapy server, without scheduling any domains"""
|
|
scrapyengine.keep_alive = keep_alive
|
|
scrapyengine.start()
|
|
if self.control_reactor:
|
|
reactor.run(installSignalHandlers=False)
|
|
|
|
def stop(self):
|
|
"""Stop the scrapy server, shutting down the execution engine"""
|
|
self.interrupted = True
|
|
scrapyengine.stop()
|
|
|
|
def _create_spider_for_request(self, request, default=None, log_none=False, \
|
|
log_multiple=False):
|
|
spider_names = spiders.find_by_request(request)
|
|
if len(spider_names) == 1:
|
|
return spiders.create(spider_names[0])
|
|
if len(spider_names) > 1 and log_multiple:
|
|
log.msg('More than one spider found for: %s' % request, log.ERROR)
|
|
if len(spider_names) == 0 and log_none:
|
|
log.msg('Could not find spider for: %s' % request, log.ERROR)
|
|
return default
|
|
|
|
def _signal_shutdown(self, signum, _):
|
|
signame = signal_names[signum]
|
|
log.msg("Received %s, shutting down gracefully. Send again to force " \
|
|
"unclean shutdown" % signame, level=log.INFO)
|
|
reactor.callFromThread(self.stop)
|
|
install_shutdown_handlers(self._signal_kill)
|
|
|
|
def _signal_kill(self, signum, _):
|
|
signame = signal_names[signum]
|
|
log.msg('Received %s twice, forcing unclean shutdown' % signame, \
|
|
level=log.INFO)
|
|
log.log_level = log.SILENT # disable logging of confusing tracebacks
|
|
reactor.callFromThread(scrapyengine.kill)
|
|
install_shutdown_handlers(signal.SIG_IGN)
|
|
|
|
scrapymanager = ExecutionManager()
|