RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

default to multi crawler commands 

compatibility for old single crawler process
deprecated command's `crawler` property
logs about settings only shown once
This commit is contained in:
Alex Cepoi 2013-08-19 22:43:29 +02:00 committed by Daniel Graña
parent 97d197267e
commit a5ffdf9bcb
5 changed files with 64 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
scrapy/cmdline.py
View File

@ -5,7 +5,7 @@ import inspect
import pkg_resources
import scrapy
from scrapy.crawler import CrawlerProcess, MultiCrawlerProcess
from scrapy.crawler import CrawlerProcess
from scrapy.xlib import lsprofcalltree
from scrapy.command import ScrapyCommand
from scrapy.exceptions import UsageError
@ -138,14 +138,7 @@ def execute(argv=None, settings=None):
opts, args = parser.parse_args(args=argv[1:])
_run_print_help(parser, cmd.process_options, args, opts)
if cmd.multi_crawlers:
process = MultiCrawlerProcess(settings)
cmd.process = process
else:
process = CrawlerProcess(settings)
process.install()
cmd.set_crawler(process)
cmd.crawler_process = CrawlerProcess(settings)
_run_print_help(parser, _run_command, cmd, args, opts)
sys.exit(cmd.exitcode)

27
scrapy/command.py
View File

@ -3,17 +3,17 @@ Base class for Scrapy commands
"""
import os
import warnings
from optparse import OptionGroup
from twisted.python import failure
from scrapy import log
from scrapy.utils.conf import arglist_to_dict
from scrapy.exceptions import UsageError
from scrapy.exceptions import UsageError, ScrapyDeprecationWarning
class ScrapyCommand(object):
requires_project = False
multi_crawlers = False
crawler_process = None
# default settings to be used for this command instead of global defaults
default_settings = {}
@ -29,9 +29,24 @@ class ScrapyCommand(object):
@property
def crawler(self):
if not self.multi_crawlers and not self._crawler.configured:
log.start_from_crawler(self._crawler)
self._crawler.configure()
warnings.warn("Command's default `crawler` is deprecated and will be removed. "
"Use `create_crawler` method to instatiate crawlers.",
ScrapyDeprecationWarning)
if not hasattr(self, '_crawler'):
crawler = self.crawler_process.create_crawler('default')
old_start = crawler.start
self.crawler_process.started = False
def wrapped_start():
if self.crawler_process.started:
old_start()
else:
self.crawler_process.started = True
self.crawler_process.start()
crawler.start = wrapped_start
self.set_crawler(crawler)
return self._crawler

5
scrapy/commands/check.py
View File

@ -20,7 +20,6 @@ def _generate(cb):
class Command(ScrapyCommand):
requires_project = True
multi_crawlers = True
default_settings = {'LOG_ENABLED': False}
def syntax(self):
@ -59,7 +58,7 @@ class Command(ScrapyCommand):
for req in requests:
contract_reqs[spider.name].append(req.callback.__name__)
elif requests:
crawler = self.process.create_crawler(spider.name)
crawler = self.crawler_process.create_crawler(spider.name)
crawler.crawl(spider, requests)
# start checks
@ -69,7 +68,7 @@ class Command(ScrapyCommand):
for method in sorted(methods):
print ' * %s' % method
else:
self.process.start()
self.crawler_process.start()
self.results.printErrors()
def get_requests(self, spider):

52
scrapy/crawler.py
View File

@ -8,7 +8,6 @@ from scrapy.extension import ExtensionManager
from scrapy.signalmanager import SignalManager
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.misc import load_object
from scrapy.settings import overridden_settings
from scrapy import log, signals
@ -35,9 +34,8 @@ class Crawler(object):
def configure(self):
if self.configured:
return
self.configured = True
d = dict(overridden_settings(self.settings))
log.msg(format="Overridden settings: %(settings)r", settings=d, level=log.DEBUG)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.extensions = ExtensionManager.from_crawler(self)
@ -84,11 +82,15 @@ class ProcessMixin(object):
install_shutdown_handlers(self._signal_shutdown)
def start(self):
self.start_crawling()
if self.settings.getbool('DNSCACHE_ENABLED'):
reactor.installResolver(CachingThreadedResolver(reactor))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
def start_crawling(self):
raise NotImplementedError
def stop(self):
raise NotImplementedError
@ -113,31 +115,12 @@ class ProcessMixin(object):
reactor.callFromThread(self._stop_reactor)
class CrawlerProcess(Crawler, ProcessMixin):
""" A class to run a single Scrapy crawler in a process
"""
def __init__(self, *a, **kw):
Crawler.__init__(self, *a, **kw)
ProcessMixin.__init__(self, *a, **kw)
self.signals.connect(self.stop, signals.engine_stopped)
def start(self):
Crawler.start(self)
ProcessMixin.start(self)
def stop(self):
d = Crawler.stop(self)
d.addBoth(self._stop_reactor)
return d
class MultiCrawlerProcess(ProcessMixin):
class CrawlerProcess(ProcessMixin):
""" A class to run multiple scrapy crawlers in a process sequentially
"""
def __init__(self, settings):
super(MultiCrawlerProcess, self).__init__(settings)
super(CrawlerProcess, self).__init__(settings)
self.settings = settings
self.crawlers = {}
@ -145,16 +128,21 @@ class MultiCrawlerProcess(ProcessMixin):
def create_crawler(self, name):
if name not in self.crawlers:
self.crawlers[name] = Crawler(self.settings)
crawler = Crawler(self.settings)
crawler.configure()
self.crawlers[name] = crawler
return self.crawlers[name]
def start_crawler(self):
def start_crawling(self):
name, crawler = self.crawlers.popitem()
crawler.sflo = log.start_from_crawler(crawler)
if crawler.sflo:
crawler.signals.connect(crawler.sflo.stop, signals.engine_stopped)
sflo = log.start_from_crawler(crawler)
crawler.install()
crawler.signals.connect(crawler.uninstall, signals.engine_stopped)
if sflo:
crawler.signals.connect(sflo.stop, signals.engine_stopped)
crawler.signals.connect(self.check_done, signals.engine_stopped)
crawler.start()
@ -163,13 +151,13 @@ class MultiCrawlerProcess(ProcessMixin):
def check_done(self, **kwargs):
if self.crawlers and not self.stopping:
self.start_crawler()
self.start_crawling()
else:
self._stop_reactor()
def start(self):
self.start_crawler()
super(MultiCrawlerProcess, self).start()
log.scrapy_info(self.settings)
return super(CrawlerProcess, self).start()
@defer.inlineCallbacks
def stop(self):

30
scrapy/log.py
View File

@ -11,6 +11,7 @@ from twisted.python import log
import scrapy
from scrapy.utils.python import unicode_to_str
from scrapy.settings import overridden_settings
# Logging levels
DEBUG = logging.DEBUG
@ -133,17 +134,24 @@ def err(_stuff=None, _why=None, **kw):
kw.setdefault('system', 'scrapy')
log.err(_stuff, _why, **kw)
def start_from_crawler(crawler):
settings = crawler.settings
if not settings.getbool('LOG_ENABLED'):
return
def start_from_settings(settings, crawler=None):
if settings.getbool('LOG_ENABLED'):
return start(settings['LOG_FILE'], settings['LOG_LEVEL'], settings['LOG_STDOUT'],
settings['LOG_ENCODING'], crawler)
sflo = start(settings['LOG_FILE'], settings['LOG_LEVEL'], settings['LOG_STDOUT'],
settings['LOG_ENCODING'], crawler)
def scrapy_info(settings):
sflo = start_from_settings(settings)
if sflo:
msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, \
settings['BOT_NAME']))
msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, \
settings['BOT_NAME']))
msg("Optional features available: %s" % ", ".join(scrapy.optional_features),
level=DEBUG)
msg("Optional features available: %s" % ", ".join(scrapy.optional_features),
level=DEBUG)
return sflo
d = dict(overridden_settings(settings))
msg(format="Overridden settings: %(settings)r", settings=d, level=DEBUG)
sflo.stop()
def start_from_crawler(crawler, print_headers=False):
return start_from_settings(crawler.settings, crawler)