mirror of https://github.com/scrapy/scrapy.git
141 lines
4.5 KiB
Python
141 lines
4.5 KiB
Python
from __future__ import annotations
|
|
|
|
from unittest.mock import Mock
|
|
|
|
import pytest
|
|
from twisted.internet.address import IPv4Address, IPv6Address
|
|
|
|
from scrapy.resolver import CachingHostnameResolver, CachingThreadedResolver, dnscache
|
|
from scrapy.utils.defer import maybe_deferred_to_future
|
|
from scrapy.utils.test import get_crawler
|
|
from tests.utils.decorators import coroutine_test
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def reset_dnscache():
|
|
original_limit = dnscache.limit
|
|
dnscache.clear()
|
|
yield
|
|
dnscache.clear()
|
|
dnscache.limit = original_limit
|
|
|
|
|
|
def test_caching_threaded_resolver_dnscache_disabled():
|
|
crawler = get_crawler(settings_dict={"DNSCACHE_ENABLED": False})
|
|
CachingThreadedResolver.from_crawler(crawler, Mock())
|
|
assert dnscache.limit == 0
|
|
|
|
|
|
@coroutine_test
|
|
async def test_caching_threaded_resolver_getHostByName_cache_hit():
|
|
resolver = CachingThreadedResolver(Mock(), cache_size=10, timeout=5.0)
|
|
dnscache["example.com"] = "1.2.3.4"
|
|
|
|
result = await maybe_deferred_to_future(resolver.getHostByName("example.com"))
|
|
assert result == "1.2.3.4"
|
|
|
|
|
|
def test_caching_hostname_resolver_dnscache_disabled():
|
|
crawler = get_crawler(settings_dict={"DNSCACHE_ENABLED": False})
|
|
CachingHostnameResolver.from_crawler(crawler, Mock())
|
|
assert dnscache.limit == 0
|
|
|
|
|
|
def test_caching_hostname_resolver_no_addresses_not_cached():
|
|
def fake_resolve(receiver, *_):
|
|
receiver.resolutionBegan(Mock())
|
|
receiver.resolutionComplete()
|
|
return receiver
|
|
|
|
reactor = Mock()
|
|
reactor.nameResolver.resolveHostName.side_effect = fake_resolve
|
|
|
|
resolver = CachingHostnameResolver(reactor, cache_size=10)
|
|
resolver.resolveHostName(Mock(), "example.com")
|
|
|
|
assert "example.com" not in dnscache
|
|
|
|
|
|
def test_caching_hostname_resolver_cached_addresses_have_no_port():
|
|
def fake_resolve(receiver, *_):
|
|
receiver.resolutionBegan(Mock())
|
|
receiver.addressResolved(IPv4Address("TCP", "1.2.3.4", 80))
|
|
receiver.addressResolved(IPv6Address("TCP", "::1", 80))
|
|
receiver.resolutionComplete()
|
|
return receiver
|
|
|
|
reactor = Mock()
|
|
reactor.nameResolver.resolveHostName.side_effect = fake_resolve
|
|
|
|
receiver = Mock()
|
|
resolver = CachingHostnameResolver(reactor, cache_size=10)
|
|
resolver.resolveHostName(receiver, "example.com", portNumber=80)
|
|
|
|
# The port requested on a cache miss is passed through unchanged, but it is
|
|
# not part of what gets cached.
|
|
resolved_ports = [
|
|
call.args[0].port for call in receiver.addressResolved.call_args_list
|
|
]
|
|
assert resolved_ports == [80, 80]
|
|
assert [address.port for address in dnscache["example.com"]] == [0, 0]
|
|
|
|
|
|
def test_caching_hostname_resolver_cache_hit_without_port():
|
|
cached_addresses = [
|
|
IPv4Address("TCP", "1.2.3.4", 0),
|
|
IPv6Address("TCP", "::1", 0),
|
|
]
|
|
dnscache["example.com"] = cached_addresses
|
|
|
|
receiver = Mock()
|
|
resolver = CachingHostnameResolver(Mock(), cache_size=10)
|
|
resolver.resolveHostName(receiver, "example.com")
|
|
|
|
# Cached addresses already use the requested port, so they are reused as is.
|
|
resolved_addresses = [
|
|
call.args[0] for call in receiver.addressResolved.call_args_list
|
|
]
|
|
assert all(
|
|
resolved is cached
|
|
for resolved, cached in zip(resolved_addresses, cached_addresses, strict=True)
|
|
)
|
|
|
|
|
|
def test_caching_hostname_resolver_cache_hit_uses_requested_port():
|
|
dnscache["example.com"] = [
|
|
IPv4Address("TCP", "1.2.3.4", 0),
|
|
IPv6Address("TCP", "::1", 0),
|
|
]
|
|
|
|
receiver = Mock()
|
|
resolver = CachingHostnameResolver(Mock(), cache_size=10)
|
|
resolver.resolveHostName(receiver, "example.com", portNumber=443)
|
|
|
|
resolved_addresses = [
|
|
call.args[0] for call in receiver.addressResolved.call_args_list
|
|
]
|
|
assert resolved_addresses == [
|
|
IPv4Address("TCP", "1.2.3.4", 443),
|
|
IPv6Address("TCP", "::1", 443),
|
|
]
|
|
# The cached addresses must not be mutated in place.
|
|
assert [address.port for address in dnscache["example.com"]] == [0, 0]
|
|
|
|
|
|
def test_caching_hostname_resolver_dnscache_disabled_rejects_storage():
|
|
|
|
def fake_resolve(receiver, *_):
|
|
receiver.resolutionBegan(Mock())
|
|
receiver.addressResolved(IPv4Address("TCP", "1.2.3.4", 80))
|
|
receiver.resolutionComplete()
|
|
return receiver
|
|
|
|
reactor = Mock()
|
|
reactor.nameResolver.resolveHostName.side_effect = fake_resolve
|
|
|
|
resolver = CachingHostnameResolver(reactor, cache_size=0)
|
|
resolver.resolveHostName(Mock(), "example.com")
|
|
|
|
assert "example.com" not in dnscache
|
|
assert len(dnscache) == 0
|