diff --git a/tests/__init__.py b/tests/__init__.py index 12ce79fa9..bb62851dc 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,6 +5,7 @@ see https://docs.scrapy.org/en/latest/contributing.html#running-tests """ import os +import socket # ignore system-wide proxies for tests # which would send requests to a totally unsuspecting server @@ -25,6 +26,15 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data') +# In some environments accessing a non-existing host doesn't raise an +# error. In such cases we're going to skip tests which rely on it. +try: + socket.getaddrinfo('non-existing-host', 80) + NON_EXISTING_RESOLVABLE = True +except socket.gaierror: + NON_EXISTING_RESOLVABLE = False + + def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 16c9559b5..33189e9be 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -6,7 +6,7 @@ from twisted.internet import defer from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest -from tests import tests_datadir +from tests import tests_datadir, NON_EXISTING_RESOLVABLE class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @@ -109,6 +109,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_dns_failures(self): + if NON_EXISTING_RESOLVABLE: + raise unittest.SkipTest("Non-existing hosts are resolvable") url = 'www.somedomainthatdoesntexi.st' errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False) self.assertEqual(errcode, 1, out or err) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 7bda3bef2..f9ffcd6bb 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -3,6 +3,7 @@ import logging from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse +import unittest from pytest import mark from testfixtures import LogCapture @@ -17,6 +18,7 @@ from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import to_unicode +from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import MockServer from tests.spiders import ( AsyncDefAsyncioGenComplexSpider, @@ -137,6 +139,8 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_retry_dns_error(self): + if NON_EXISTING_RESOLVABLE: + raise unittest.SkipTest("Non-existing hosts are resolvable") crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as log: # try to fetch the homepage of a non-existent domain diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 72f52121e..883960084 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -4,7 +4,7 @@ import shutil import sys import tempfile from typing import Optional, Type -from unittest import mock +from unittest import mock, SkipTest from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal @@ -32,6 +32,7 @@ from scrapy.spiders import Spider from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto +from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import ( Echo, ForeverTakingResource, @@ -791,6 +792,8 @@ class Http11ProxyTestCase(HttpProxyTestCase): @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): """ Test TunnelingTCP4ClientEndpoint """ + if NON_EXISTING_RESOLVABLE: + raise SkipTest("Non-existing hosts are resolvable") http_proxy = self.getURL('') domain = 'https://no-such-domain.nosuch' request = Request(