RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: failed tests

This commit is contained in:
Laerte Pereira 2022-09-06 01:15:41 -03:00
parent 52d93490f5
commit f4bcc3e67d
No known key found for this signature in database
GPG Key ID: BB69B5C731BB8A80
2 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
tests/test_crawler.py
View File

@ -23,6 +23,8 @@ from scrapy.utils.test import get_crawler
from scrapy.extensions.throttle import AutoThrottle
from scrapy.extensions import telnet
from scrapy.utils.test import get_testenv
from pkg_resources import parse_version
from w3lib import __version__ as w3lib_version
from tests.mockserver import MockServer
@ -381,10 +383,13 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
def test_ipv6_default_name_resolver(self):
log = self.run_script('default_name_resolver.py')
self.assertIn('Spider closed (finished)', log)
self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
self.assertIn(
"twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",
log)
if parse_version(w3lib_version) < parse_version("2.0.0"):
self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
self.assertIn(
"twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",
log)
else:
self.assertIn("ValueError: invalid hostname:", log)
def test_caching_hostname_resolver_ipv6(self):
log = self.run_script("caching_hostname_resolver_ipv6.py")

12
tests/test_spider.py
View File

@ -22,6 +22,8 @@ from scrapy.spiders import (
from scrapy.linkextractors import LinkExtractor
from scrapy.utils.test import get_crawler
from tests import get_testdata
from pkg_resources import parse_version
from w3lib import __version__ as w3lib_version
class SpiderTest(unittest.TestCase):
@ -360,10 +362,12 @@ class CrawlSpiderTest(SpiderTest):
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
self.assertEqual([r.url for r in output],
['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
'http://EXAMPLE.ORG/ABOUT.HTML',
'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
urls = ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
'http://EXAMPLE.ORG/ABOUT.HTML',
'http://EXAMPLE.ORG/NOFOLLOW.HTML']
if parse_version(w3lib_version) >= parse_version('2.0.0'):
urls = list(map(lambda u: u.replace("EXAMPLE.ORG", "example.org"), urls))
self.assertEqual([r.url for r in output], urls)
def test_process_request_instance_method_with_response(self):