mirror of https://github.com/scrapy/scrapy.git
Improve coverage resolver (#7642)
* Improve test coverage for resolver.py * Make the Twisted code more readable
This commit is contained in:
parent
d2842a205c
commit
6393858c7e
|
|
@ -75,7 +75,7 @@ class HostResolution:
|
||||||
def __init__(self, name: str):
|
def __init__(self, name: str):
|
||||||
self.name: str = name
|
self.name: str = name
|
||||||
|
|
||||||
def cancel(self) -> None:
|
def cancel(self) -> None: # pragma: no cover
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,55 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from unittest.mock import Mock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from scrapy.resolver import CachingHostnameResolver, CachingThreadedResolver, dnscache
|
||||||
|
from scrapy.utils.defer import maybe_deferred_to_future
|
||||||
|
from scrapy.utils.test import get_crawler
|
||||||
|
from tests.utils.decorators import coroutine_test
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def reset_dnscache():
|
||||||
|
original_limit = dnscache.limit
|
||||||
|
dnscache.clear()
|
||||||
|
yield
|
||||||
|
dnscache.clear()
|
||||||
|
dnscache.limit = original_limit
|
||||||
|
|
||||||
|
|
||||||
|
def test_caching_threaded_resolver_dnscache_disabled():
|
||||||
|
crawler = get_crawler(settings_dict={"DNSCACHE_ENABLED": False})
|
||||||
|
CachingThreadedResolver.from_crawler(crawler, Mock())
|
||||||
|
assert dnscache.limit == 0
|
||||||
|
|
||||||
|
|
||||||
|
@coroutine_test
|
||||||
|
async def test_caching_threaded_resolver_getHostByName_cache_hit():
|
||||||
|
resolver = CachingThreadedResolver(Mock(), cache_size=10, timeout=5.0)
|
||||||
|
dnscache["example.com"] = "1.2.3.4"
|
||||||
|
|
||||||
|
result = await maybe_deferred_to_future(resolver.getHostByName("example.com"))
|
||||||
|
assert result == "1.2.3.4"
|
||||||
|
|
||||||
|
|
||||||
|
def test_caching_hostname_resolver_dnscache_disabled():
|
||||||
|
crawler = get_crawler(settings_dict={"DNSCACHE_ENABLED": False})
|
||||||
|
CachingHostnameResolver.from_crawler(crawler, Mock())
|
||||||
|
assert dnscache.limit == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_caching_hostname_resolver_no_addresses_not_cached():
|
||||||
|
def fake_resolve(receiver, *_):
|
||||||
|
receiver.resolutionBegan(Mock())
|
||||||
|
receiver.resolutionComplete()
|
||||||
|
return receiver
|
||||||
|
|
||||||
|
reactor = Mock()
|
||||||
|
reactor.nameResolver.resolveHostName.side_effect = fake_resolve
|
||||||
|
|
||||||
|
resolver = CachingHostnameResolver(reactor, cache_size=10)
|
||||||
|
resolver.resolveHostName(Mock(), "example.com")
|
||||||
|
|
||||||
|
assert "example.com" not in dnscache
|
||||||
Loading…
Reference in New Issue