From f4bcc3e67de0896999be48d1b0fcede0be0d69e4 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 6 Sep 2022 01:15:41 -0300 Subject: [PATCH 1/3] fix: failed tests --- tests/test_crawler.py | 13 +++++++++---- tests/test_spider.py | 12 ++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 4e9ef7740..4e599d69c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -23,6 +23,8 @@ from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version from tests.mockserver import MockServer @@ -381,10 +383,13 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) - self.assertIn( - "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", - log) + if parse_version(w3lib_version) < parse_version("2.0.0"): + self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) + self.assertIn( + "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", + log) + else: + self.assertIn("ValueError: invalid hostname:", log) def test_caching_hostname_resolver_ipv6(self): log = self.run_script("caching_hostname_resolver_ipv6.py") diff --git a/tests/test_spider.py b/tests/test_spider.py index 689349999..7b36304b1 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -22,6 +22,8 @@ from scrapy.spiders import ( from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler from tests import get_testdata +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version class SpiderTest(unittest.TestCase): @@ -360,10 +362,12 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) + urls = ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', + 'http://EXAMPLE.ORG/ABOUT.HTML', + 'http://EXAMPLE.ORG/NOFOLLOW.HTML'] + if parse_version(w3lib_version) >= parse_version('2.0.0'): + urls = list(map(lambda u: u.replace("EXAMPLE.ORG", "example.org"), urls)) + self.assertEqual([r.url for r in output], urls) def test_process_request_instance_method_with_response(self): From 1289422284991c09cbee8cea17bef837b3efc31b Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 6 Sep 2022 08:17:58 -0300 Subject: [PATCH 2/3] chore: Skip `test_ipv6_default_name_resolver` test if w3lib version >= 2.0.0 --- tests/test_crawler.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 4e599d69c..e2a14be55 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -380,16 +380,15 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(parse_version(w3lib_version) >= parse_version("2.0.0"), + reason='w3lib 2.0.0 and later do not allow invalid domains.') def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') self.assertIn('Spider closed (finished)', log) - if parse_version(w3lib_version) < parse_version("2.0.0"): - self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) - self.assertIn( - "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", - log) - else: - self.assertIn("ValueError: invalid hostname:", log) + self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) + self.assertIn( + "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", + log) def test_caching_hostname_resolver_ipv6(self): log = self.run_script("caching_hostname_resolver_ipv6.py") From 582a6bf6dbcb01da40cbec6b51269add2db39cf1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 6 Sep 2022 10:03:18 -0300 Subject: [PATCH 3/3] refactor: Use `safe_url_string` to standardize url output --- tests/test_spider.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 7b36304b1..e1527620f 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -22,8 +22,7 @@ from scrapy.spiders import ( from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler from tests import get_testdata -from pkg_resources import parse_version -from w3lib import __version__ as w3lib_version +from w3lib.url import safe_url_string class SpiderTest(unittest.TestCase): @@ -362,12 +361,10 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - urls = ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML'] - if parse_version(w3lib_version) >= parse_version('2.0.0'): - urls = list(map(lambda u: u.replace("EXAMPLE.ORG", "example.org"), urls)) - self.assertEqual([r.url for r in output], urls) + self.assertEqual([r.url for r in output], + [safe_url_string('http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML'), + safe_url_string('http://EXAMPLE.ORG/ABOUT.HTML'), + safe_url_string('http://EXAMPLE.ORG/NOFOLLOW.HTML')]) def test_process_request_instance_method_with_response(self):