From a5ffdf9bcb2fb25204378eea821047b540b1cab6 Mon Sep 17 00:00:00 2001 From: Alex Cepoi Date: Mon, 19 Aug 2013 22:43:29 +0200 Subject: [PATCH] default to multi crawler commands compatibility for old single crawler process deprecated command's `crawler` property logs about settings only shown once --- scrapy/cmdline.py | 11 ++------- scrapy/command.py | 27 ++++++++++++++++----- scrapy/commands/check.py | 5 ++-- scrapy/crawler.py | 52 ++++++++++++++++------------------------ scrapy/log.py | 30 ++++++++++++++--------- 5 files changed, 64 insertions(+), 61 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 943bb43a9..8c9ea120e 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -5,7 +5,7 @@ import inspect import pkg_resources import scrapy -from scrapy.crawler import CrawlerProcess, MultiCrawlerProcess +from scrapy.crawler import CrawlerProcess from scrapy.xlib import lsprofcalltree from scrapy.command import ScrapyCommand from scrapy.exceptions import UsageError @@ -138,14 +138,7 @@ def execute(argv=None, settings=None): opts, args = parser.parse_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) - if cmd.multi_crawlers: - process = MultiCrawlerProcess(settings) - cmd.process = process - else: - process = CrawlerProcess(settings) - process.install() - cmd.set_crawler(process) - + cmd.crawler_process = CrawlerProcess(settings) _run_print_help(parser, _run_command, cmd, args, opts) sys.exit(cmd.exitcode) diff --git a/scrapy/command.py b/scrapy/command.py index 87b0cff26..b7c16b890 100644 --- a/scrapy/command.py +++ b/scrapy/command.py @@ -3,17 +3,17 @@ Base class for Scrapy commands """ import os +import warnings from optparse import OptionGroup from twisted.python import failure -from scrapy import log from scrapy.utils.conf import arglist_to_dict -from scrapy.exceptions import UsageError +from scrapy.exceptions import UsageError, ScrapyDeprecationWarning class ScrapyCommand(object): requires_project = False - multi_crawlers = False + crawler_process = None # default settings to be used for this command instead of global defaults default_settings = {} @@ -29,9 +29,24 @@ class ScrapyCommand(object): @property def crawler(self): - if not self.multi_crawlers and not self._crawler.configured: - log.start_from_crawler(self._crawler) - self._crawler.configure() + warnings.warn("Command's default `crawler` is deprecated and will be removed. " + "Use `create_crawler` method to instatiate crawlers.", + ScrapyDeprecationWarning) + + if not hasattr(self, '_crawler'): + crawler = self.crawler_process.create_crawler('default') + + old_start = crawler.start + self.crawler_process.started = False + def wrapped_start(): + if self.crawler_process.started: + old_start() + else: + self.crawler_process.started = True + self.crawler_process.start() + crawler.start = wrapped_start + + self.set_crawler(crawler) return self._crawler diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 297f59b3a..54d90b879 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -20,7 +20,6 @@ def _generate(cb): class Command(ScrapyCommand): requires_project = True - multi_crawlers = True default_settings = {'LOG_ENABLED': False} def syntax(self): @@ -59,7 +58,7 @@ class Command(ScrapyCommand): for req in requests: contract_reqs[spider.name].append(req.callback.__name__) elif requests: - crawler = self.process.create_crawler(spider.name) + crawler = self.crawler_process.create_crawler(spider.name) crawler.crawl(spider, requests) # start checks @@ -69,7 +68,7 @@ class Command(ScrapyCommand): for method in sorted(methods): print ' * %s' % method else: - self.process.start() + self.crawler_process.start() self.results.printErrors() def get_requests(self, spider): diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 4c650c41c..9b6f7bc25 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -8,7 +8,6 @@ from scrapy.extension import ExtensionManager from scrapy.signalmanager import SignalManager from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object -from scrapy.settings import overridden_settings from scrapy import log, signals @@ -35,9 +34,8 @@ class Crawler(object): def configure(self): if self.configured: return + self.configured = True - d = dict(overridden_settings(self.settings)) - log.msg(format="Overridden settings: %(settings)r", settings=d, level=log.DEBUG) lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) self.extensions = ExtensionManager.from_crawler(self) @@ -84,11 +82,15 @@ class ProcessMixin(object): install_shutdown_handlers(self._signal_shutdown) def start(self): + self.start_crawling() if self.settings.getbool('DNSCACHE_ENABLED'): reactor.installResolver(CachingThreadedResolver(reactor)) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call + def start_crawling(self): + raise NotImplementedError + def stop(self): raise NotImplementedError @@ -113,31 +115,12 @@ class ProcessMixin(object): reactor.callFromThread(self._stop_reactor) -class CrawlerProcess(Crawler, ProcessMixin): - """ A class to run a single Scrapy crawler in a process - """ - - def __init__(self, *a, **kw): - Crawler.__init__(self, *a, **kw) - ProcessMixin.__init__(self, *a, **kw) - self.signals.connect(self.stop, signals.engine_stopped) - - def start(self): - Crawler.start(self) - ProcessMixin.start(self) - - def stop(self): - d = Crawler.stop(self) - d.addBoth(self._stop_reactor) - return d - - -class MultiCrawlerProcess(ProcessMixin): +class CrawlerProcess(ProcessMixin): """ A class to run multiple scrapy crawlers in a process sequentially """ def __init__(self, settings): - super(MultiCrawlerProcess, self).__init__(settings) + super(CrawlerProcess, self).__init__(settings) self.settings = settings self.crawlers = {} @@ -145,16 +128,21 @@ class MultiCrawlerProcess(ProcessMixin): def create_crawler(self, name): if name not in self.crawlers: - self.crawlers[name] = Crawler(self.settings) + crawler = Crawler(self.settings) + crawler.configure() + + self.crawlers[name] = crawler return self.crawlers[name] - def start_crawler(self): + def start_crawling(self): name, crawler = self.crawlers.popitem() - crawler.sflo = log.start_from_crawler(crawler) - if crawler.sflo: - crawler.signals.connect(crawler.sflo.stop, signals.engine_stopped) + sflo = log.start_from_crawler(crawler) + crawler.install() + crawler.signals.connect(crawler.uninstall, signals.engine_stopped) + if sflo: + crawler.signals.connect(sflo.stop, signals.engine_stopped) crawler.signals.connect(self.check_done, signals.engine_stopped) crawler.start() @@ -163,13 +151,13 @@ class MultiCrawlerProcess(ProcessMixin): def check_done(self, **kwargs): if self.crawlers and not self.stopping: - self.start_crawler() + self.start_crawling() else: self._stop_reactor() def start(self): - self.start_crawler() - super(MultiCrawlerProcess, self).start() + log.scrapy_info(self.settings) + return super(CrawlerProcess, self).start() @defer.inlineCallbacks def stop(self): diff --git a/scrapy/log.py b/scrapy/log.py index 00ef5e5bc..2f2b47a92 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -11,6 +11,7 @@ from twisted.python import log import scrapy from scrapy.utils.python import unicode_to_str +from scrapy.settings import overridden_settings # Logging levels DEBUG = logging.DEBUG @@ -133,17 +134,24 @@ def err(_stuff=None, _why=None, **kw): kw.setdefault('system', 'scrapy') log.err(_stuff, _why, **kw) -def start_from_crawler(crawler): - settings = crawler.settings - if not settings.getbool('LOG_ENABLED'): - return +def start_from_settings(settings, crawler=None): + if settings.getbool('LOG_ENABLED'): + return start(settings['LOG_FILE'], settings['LOG_LEVEL'], settings['LOG_STDOUT'], + settings['LOG_ENCODING'], crawler) - sflo = start(settings['LOG_FILE'], settings['LOG_LEVEL'], settings['LOG_STDOUT'], - settings['LOG_ENCODING'], crawler) +def scrapy_info(settings): + sflo = start_from_settings(settings) + if sflo: + msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, \ + settings['BOT_NAME'])) - msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, \ - settings['BOT_NAME'])) - msg("Optional features available: %s" % ", ".join(scrapy.optional_features), - level=DEBUG) + msg("Optional features available: %s" % ", ".join(scrapy.optional_features), + level=DEBUG) - return sflo + d = dict(overridden_settings(settings)) + msg(format="Overridden settings: %(settings)r", settings=d, level=DEBUG) + + sflo.stop() + +def start_from_crawler(crawler, print_headers=False): + return start_from_settings(crawler.settings, crawler)