From d3f82aa4d50a214343d9d2cd78bf782d53deb1c3 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 7 Sep 2022 12:52:41 +0500 Subject: [PATCH] Merge pull request #5617 from Laerte/fix/tests-w3lib Fix failed tests related to w3lib --- tests/test_crawler.py | 4 ++++ tests/test_spider.py | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 4801ad871..0cf23ee04 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -21,6 +21,8 @@ from scrapy.utils.misc import load_object from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version from tests.mockserver import MockServer @@ -369,6 +371,8 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(parse_version(w3lib_version) >= parse_version("2.0.0"), + reason='w3lib 2.0.0 and later do not allow invalid domains.') def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') self.assertIn('Spider closed (finished)', log) diff --git a/tests/test_spider.py b/tests/test_spider.py index 689349999..e1527620f 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -22,6 +22,7 @@ from scrapy.spiders import ( from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler from tests import get_testdata +from w3lib.url import safe_url_string class SpiderTest(unittest.TestCase): @@ -361,9 +362,9 @@ class CrawlSpiderTest(SpiderTest): self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) self.assertEqual([r.url for r in output], - ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) + [safe_url_string('http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML'), + safe_url_string('http://EXAMPLE.ORG/ABOUT.HTML'), + safe_url_string('http://EXAMPLE.ORG/NOFOLLOW.HTML')]) def test_process_request_instance_method_with_response(self):