diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index cf9d459d0..f39da779f 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,4 +1,3 @@ -import socket import random import warnings from time import time @@ -8,12 +7,10 @@ from functools import partial from twisted.internet import reactor, defer from twisted.python.failure import Failure -from scrapy.utils.python import setattr_default from scrapy.utils.defer import mustbe_deferred from scrapy.utils.signal import send_catch_log -from scrapy.utils.reactor import CallLaterOnce from scrapy.utils.httpobj import urlparse_cached -from scrapy.resolver import gethostbyname +from scrapy.resolver import dnscache from scrapy import signals from scrapy import log from .middleware import DownloaderMiddlewareManager @@ -95,14 +92,12 @@ class Downloader(object): return len(self.active) >= self.total_concurrency def _get_slot(self, request, spider): - key = urlparse_cached(request).hostname + key = urlparse_cached(request).hostname or '' + if self.ip_concurrency: + key = dnscache.get(key, key) if key not in self.slots: if self.ip_concurrency: concurrency = self.ip_concurrency - try: - key = gethostbyname(key) - except socket.error: # resolution error - pass else: concurrency = self.domain_concurrency concurrency, delay = _get_concurrency_delay(concurrency, spider, self.settings) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 8ccd400d5..6ba89ac94 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -10,11 +10,6 @@ from scrapy.utils.datatypes import LocalCache dnscache = LocalCache(10000) -def gethostbyname(hostname): - if hostname not in dnscache: - dnscache[hostname] = socket.gethostbyname(hostname) - return dnscache[hostname] - class CachingThreadedResolver(ThreadedResolver): def getHostByName(self, name, timeout = (1, 3, 11, 45)):