RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Name resolvers: install_on_reactor as instance method

This commit is contained in:
Eugenio Lacuesta 2020-01-16 18:01:18 -03:00
parent f45b4c7f8d
commit 3cfa73b8b1
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
2 changed files with 7 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/crawler.py
View File

@ -21,7 +21,7 @@ from scrapy.utils.log import (
log_scrapy_info,
LogCounterHandler,
)
from scrapy.utils.misc import load_object
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
@ -305,7 +305,8 @@ class CrawlerProcess(CrawlerRunner):
d.addBoth(self._stop_reactor)
resolver_class = load_object(self.settings["DNS_RESOLVER"])
resolver_class.install_on_reactor(reactor, crawler=self)
resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
resolver.install_on_reactor(reactor)
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)

13
scrapy/resolver.py
View File

@ -4,7 +4,6 @@ from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver,
from zope.interface.declarations import implementer, provider
from scrapy.utils.datatypes import LocalCache
from scrapy.utils.misc import create_instance
# TODO: cache misses
@ -30,10 +29,8 @@ class CachingThreadedResolver(ThreadedResolver):
cache_size = 0
return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT'))
@classmethod
def install_on_reactor(cls, reactor, crawler):
resolver = create_instance(cls, None, crawler, reactor=reactor)
reactor.installResolver(resolver)
def install_on_reactor(self, reactor):
reactor.installResolver(self)
def getHostByName(self, name, timeout=None):
if name in dnscache:
@ -72,10 +69,8 @@ class CachingHostnameResolver:
cache_size = 0
return cls(reactor, cache_size)
@classmethod
def install_on_reactor(cls, reactor, crawler):
resolver = create_instance(cls, None, crawler, reactor=reactor)
reactor.installNameResolver(resolver)
def install_on_reactor(self, reactor):
reactor.installNameResolver(self)
def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
addressTypes=None, transportSemantics='TCP'):