RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Flake8: Remove E128 (tests, part 1)

This commit is contained in:
Eugenio Lacuesta 2020-05-06 18:56:14 -03:00
parent 2851f641f2
commit 9c6d13075a
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
11 changed files with 80 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
pytest.ini
View File

@ -172,37 +172,37 @@ flake8-ignore =
tests/spiders.py E501
tests/test_closespider.py E501
tests/test_command_fetch.py E501
tests/test_command_parse.py E501 E128
tests/test_command_shell.py E501 E128
tests/test_commands.py E128 E501
tests/test_contracts.py E501 E128
tests/test_command_parse.py E501
tests/test_command_shell.py E501
tests/test_commands.py E501
tests/test_contracts.py E501
tests/test_crawl.py E501 E741
tests/test_crawler.py F841 E501
tests/test_dependencies.py F841 E501
tests/test_downloader_handlers.py E128 E501
tests/test_downloader_handlers.py E501
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E741 E501 E128
tests/test_downloadermiddleware_cookies.py E741 E501
tests/test_downloadermiddleware_defaultheaders.py E501
tests/test_downloadermiddleware_downloadtimeout.py E501
tests/test_downloadermiddleware_httpcache.py E501
tests/test_downloadermiddleware_httpcompression.py E501
tests/test_downloadermiddleware_decompression.py E501
tests/test_downloadermiddleware_httpproxy.py E501 E128
tests/test_downloadermiddleware_redirect.py E501 E128
tests/test_downloadermiddleware_retry.py E501 E128
tests/test_downloadermiddleware_httpproxy.py E501
tests/test_downloadermiddleware_redirect.py E501
tests/test_downloadermiddleware_retry.py E501
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
tests/test_dupefilters.py E501 E741 E128
tests/test_engine.py E501 E128
tests/test_exporters.py E501 E128
tests/test_dupefilters.py E501 E741
tests/test_engine.py E501
tests/test_exporters.py E501
tests/test_extension_telnet.py F841
tests/test_feedexport.py E501 F841
tests/test_http_cookies.py E501
tests/test_http_headers.py E501
tests/test_http_request.py E402 E501 E128 E128
tests/test_http_response.py E501 E128
tests/test_item.py E128 F841
tests/test_http_request.py E402 E501
tests/test_http_response.py E501
tests/test_link.py E501
tests/test_linkextractors.py E501 E128
tests/test_loader.py E501 E741 E128 E117

4
tests/test_command_parse.py
View File

@ -142,8 +142,8 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
@defer.inlineCallbacks
def test_request_without_meta(self):
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'-c', 'parse_request_without_meta',
'--nolinks',
'-c', 'parse_request_without_meta',
'--nolinks',
self.url('/html')])
self.assertIn("DEBUG: It Works!", _textmode(stderr))

6
tests/test_command_shell.py
View File

@ -101,15 +101,13 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks
def test_local_nofile(self):
filepath = 'file:///tests/sample_data/test_site/nothinghere.html'
errcode, out, err = yield self.execute([filepath, '-c', 'item'],
check_code=False)
errcode, out, err = yield self.execute([filepath, '-c', 'item'], check_code=False)
self.assertEqual(errcode, 1, out or err)
self.assertIn(b'No such file or directory', err)
@defer.inlineCallbacks
def test_dns_failures(self):
url = 'www.somedomainthatdoesntexi.st'
errcode, out, err = yield self.execute([url, '-c', 'item'],
check_code=False)
errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False)
self.assertEqual(errcode, 1, out or err)
self.assertIn(b'DNS lookup failed', err)

3
tests/test_contracts.py
View File

@ -232,7 +232,8 @@ class ContractsManagerTest(unittest.TestCase):
# extract contracts correctly
contracts = self.conman.extract_contracts(spider.returns_request)
self.assertEqual(len(contracts), 2)
self.assertEqual(frozenset(type(x) for x in contracts),
self.assertEqual(
frozenset(type(x) for x in contracts),
frozenset([UrlContract, ReturnsContract]))
# returns request for valid method

8
tests/test_downloader_handlers.py
View File

@ -853,8 +853,7 @@ class S3TestCase(unittest.TestCase):
def test_request_signing4(self):
# fetches the access control policy sub-resource for the 'johnsmith' bucket.
date = 'Tue, 27 Mar 2007 19:44:46 +0000'
req = Request('s3://johnsmith/?acl',
method='GET', headers={'Date': date})
req = Request('s3://johnsmith/?acl', method='GET', headers={'Date': date})
with self._mocked_date(date):
httpreq = self.download_request(req, self.spider)
self.assertEqual(httpreq.headers['Authorization'],
@ -879,8 +878,9 @@ class S3TestCase(unittest.TestCase):
with self._mocked_date(date):
httpreq = self.download_request(req, self.spider)
# botocore does not override Date with x-amz-date
self.assertEqual(httpreq.headers['Authorization'],
b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
self.assertEqual(
httpreq.headers['Authorization'],
b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
def test_request_signing6(self):
# uploads an object to a CNAME style virtual hosted bucket with metadata.

7
tests/test_downloadermiddleware_httpproxy.py
View File

@ -43,8 +43,11 @@ class TestHttpProxyMiddleware(TestCase):
os.environ.pop('file_proxy', None)
mw = HttpProxyMiddleware()
for url, proxy in [('http://e.com', http_proxy),
('https://e.com', https_proxy), ('file://tmp/a', None)]:
for url, proxy in [
('http://e.com', http_proxy),
('https://e.com', https_proxy),
('file://tmp/a', None),
]:
req = Request(url)
assert mw.process_request(req, spider) is None
self.assertEqual(req.url, url)

2
tests/test_downloadermiddleware_redirect.py
View File

@ -72,7 +72,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
url = 'http://www.example.com/302'
url2 = 'http://www.example.com/redirected2'
req = Request(url, method='POST', body='test',
headers={'Content-Type': 'text/plain', 'Content-length': '4'})
headers={'Content-Type': 'text/plain', 'Content-length': '4'})
rsp = Response(url, headers={'Location': url2}, status=302)
req2 = self.mw.process_response(req, rsp, self.spider)

14
tests/test_downloadermiddleware_retry.py
View File

@ -81,9 +81,17 @@ class RetryTest(unittest.TestCase):
assert self.crawler.stats.get_value('retry/count') == 2
def test_twistederrors(self):
exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError,
DNSLookupError, ConnectionRefusedError, ConnectionDone,
ConnectError, ConnectionLost, ResponseFailed]
exceptions = [
ConnectError,
ConnectionDone,
ConnectionLost,
ConnectionRefusedError,
defer.TimeoutError,
DNSLookupError,
ResponseFailed,
TCPTimedOutError,
TimeoutError,
]
for exc in exceptions:
req = Request('http://www.scrapytest.org/%s' % exc.__name__)

31
tests/test_dupefilters.py
View File

@ -177,10 +177,13 @@ class RFPDupeFilterTest(unittest.TestCase):
dupefilter.log(r2, spider)
assert crawler.stats.get_value('dupefilter/filtered') == 2
l.check_present(('scrapy.dupefilters', 'DEBUG',
('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
' - no more duplicates will be shown'
' (see DUPEFILTER_DEBUG to show all duplicates)')))
l.check_present(
(
'scrapy.dupefilters',
'DEBUG',
'Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)'
)
)
dupefilter.close('finished')
@ -203,11 +206,19 @@ class RFPDupeFilterTest(unittest.TestCase):
dupefilter.log(r2, spider)
assert crawler.stats.get_value('dupefilter/filtered') == 2
l.check_present(('scrapy.dupefilters', 'DEBUG',
('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
' (referer: None)')))
l.check_present(('scrapy.dupefilters', 'DEBUG',
('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
' (referer: http://scrapytest.org/INDEX.html)')))
l.check_present(
(
'scrapy.dupefilters',
'DEBUG',
'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'
)
)
l.check_present(
(
'scrapy.dupefilters',
'DEBUG',
'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: http://scrapytest.org/INDEX.html)'
)
)
dupefilter.close('finished')

18
tests/test_http_request.py
View File

@ -415,8 +415,7 @@ class FormRequestTest(RequestTest):
# using multiples values for a single key
data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}
r3 = self.request_class("http://www.example.com", formdata=data)
self.assertQueryEqual(r3.body,
b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
def test_from_response_post(self):
response = _buildresponse(
@ -426,8 +425,7 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="test2" value="xxx">
</form>""",
url="http://www.example.com/this/list.html")
req = self.request_class.from_response(response,
formdata={'one': ['two', 'three'], 'six': 'seven'})
req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
self.assertEqual(req.method, 'POST')
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@ -446,8 +444,7 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="test2" value="xxx \xc2\xb5">
</form>""",
url="http://www.example.com/this/list.html")
req = self.request_class.from_response(response,
formdata={'one': ['two', 'three'], 'six': 'seven'})
req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
self.assertEqual(req.method, 'POST')
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@ -468,8 +465,7 @@ class FormRequestTest(RequestTest):
url="http://www.example.com/this/list.html",
encoding='latin1',
)
req = self.request_class.from_response(response,
formdata={'one': ['two', 'three'], 'six': 'seven'})
req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
self.assertEqual(req.method, 'POST')
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@ -488,8 +484,7 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="test2" value="xxx µ">
</form>""",
url="http://www.example.com/this/list.html")
req = self.request_class.from_response(response,
formdata={'one': ['two', 'three'], 'six': 'seven'})
req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
self.assertEqual(req.method, 'POST')
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
@ -549,8 +544,7 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="test2" value="xxx">
</form>""",
url="http://www.example.com/this/list.html")
r1 = self.request_class.from_response(response,
formdata={'one': ['two', 'three'], 'six': 'seven'})
r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
self.assertEqual(r1.method, 'GET')
self.assertEqual(urlparse(r1.url).hostname, "www.example.com")
self.assertEqual(urlparse(r1.url).path, "/this/get.php")

20
tests/test_http_response.py
View File

@ -485,8 +485,10 @@ class TextResponseTest(BaseResponseTest):
response.xpath("normalize-space(//p[@class=\"content\"])").getall(),
)
self.assertEqual(
response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",
pclass="content", pcount=1).getall(),
response.xpath(
"//title[count(following::p[@class=$pclass])=$pcount]/text()",
pclass="content", pcount=1,
).getall(),
response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(),
)
@ -566,12 +568,14 @@ class TextResponseTest(BaseResponseTest):
'http://example.com',
body=b'''<html><body><a href=" foo\n">click me</a></body></html>'''
)
self._assert_followed_url(resp.css('a')[0],
'http://example.com/foo',
response=resp)
self._assert_followed_url(resp.css('a::attr(href)')[0],
'http://example.com/foo',
response=resp)
self._assert_followed_url(
resp.css('a')[0],
'http://example.com/foo',
response=resp)
self._assert_followed_url(
resp.css('a::attr(href)')[0],
'http://example.com/foo',
response=resp)
def test_follow_encoding(self):
resp1 = self.response_class(