CrawlerProcess cleanup

* remove unneeded lambda;
* extract _get_dns_resolver method and format code to pep8.

(cherry picked from commit 33d145e2f5)
This commit is contained in:
Mikhail Korobov 2015-06-06 02:49:39 +05:00 committed by Julia Medina
parent 2677f70fc3
commit 3b0391b8f2
1 changed files with 13 additions and 4 deletions

View File

@ -242,16 +242,25 @@ class CrawlerProcess(CrawlerRunner):
# Don't start the reactor if the deferreds are already fired
if d.called:
return
d.addBoth(lambda _: self._stop_reactor())
d.addBoth(self._stop_reactor)
cache_size = self.settings.getint('DNSCACHE_SIZE') if self.settings.getbool('DNSCACHE_ENABLED') else 0
reactor.installResolver(CachingThreadedResolver(reactor, cache_size,
self.settings.getfloat('DNS_TIMEOUT')))
reactor.installResolver(self._get_dns_resolver())
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
def _get_dns_resolver(self):
if self.settings.getbool('DNSCACHE_ENABLED'):
cache_size = self.settings.getint('DNSCACHE_SIZE')
else:
cache_size = 0
return CachingThreadedResolver(
reactor=reactor,
cache_size=cache_size,
timeout=self.settings.getfloat('DNS_TIMEOUT')
)
def _stop_reactor(self, _=None):
try:
reactor.stop()