RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4542 from elacuesta/flake8-remove-e122-e124 

Flake8: remove E122 and E124
This commit is contained in:
Mikhail Korobov 2020-05-07 15:05:01 +05:00 committed by GitHub
parent 2da952b928 cc23d1cb58
commit a18bfccac2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 125 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
pytest.ini
View File

@ -88,7 +88,7 @@ flake8-ignore =
scrapy/http/request/form.py E501
scrapy/http/request/json_request.py E501
scrapy/http/response/__init__.py E501 E128
scrapy/http/response/text.py E501 E128 E124
scrapy/http/response/text.py E501 E128
# scrapy/linkextractors
scrapy/linkextractors/__init__.py E501 E402
scrapy/linkextractors/lxmlhtml.py E501
@ -181,7 +181,7 @@ flake8-ignore =
tests/test_crawl.py E501 E741
tests/test_crawler.py F841 E501
tests/test_dependencies.py F841 E501
tests/test_downloader_handlers.py E124 E128 E501 E126
tests/test_downloader_handlers.py E128 E501 E126
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
@ -195,9 +195,9 @@ flake8-ignore =
tests/test_downloadermiddleware_retry.py E501 E128 E126
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
tests/test_dupefilters.py E501 E741 E128 E124
tests/test_dupefilters.py E501 E741 E128
tests/test_engine.py E501 E128
tests/test_exporters.py E501 E128 E124
tests/test_exporters.py E501 E128
tests/test_extension_telnet.py F841
tests/test_feedexport.py E501 F841
tests/test_http_cookies.py E501
@ -206,9 +206,9 @@ flake8-ignore =
tests/test_http_response.py E501 E128
tests/test_item.py E128 F841
tests/test_link.py E501
tests/test_linkextractors.py E501 E128 E124
tests/test_linkextractors.py E501 E128
tests/test_loader.py E501 E741 E128 E117
tests/test_logformatter.py E128 E501 E122
tests/test_logformatter.py E128 E501
tests/test_mail.py E128 E501
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E501 E128 E126
@ -226,7 +226,7 @@ flake8-ignore =
tests/test_spidermiddleware_httperror.py E128 E501 E121
tests/test_spidermiddleware_offsite.py E501 E128 E111
tests/test_spidermiddleware_output_chain.py E501
tests/test_spidermiddleware_referer.py E501 F841 E124 E501 E121
tests/test_spidermiddleware_referer.py E501 F841 E501 E121
tests/test_squeues.py E501 E741
tests/test_utils_asyncio.py E501
tests/test_utils_conf.py E501 E128
@ -242,9 +242,9 @@ flake8-ignore =
tests/test_utils_request.py E501 E128
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841
tests/test_utils_sitemap.py E128 E501 E124
tests/test_utils_sitemap.py E128 E501
tests/test_utils_url.py E501 E501 E126
tests/test_webclient.py E501 E128 E122 E402 E126
tests/test_webclient.py E501 E128 E402 E126
tests/test_cmdline/__init__.py E501
tests/test_settings/__init__.py E501 E128
tests/test_spiderloader/__init__.py E128 E501

3
tests/test_dupefilters.py
View File

@ -197,8 +197,7 @@ class RFPDupeFilterTest(unittest.TestCase):
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/index.html',
headers={'Referer': 'http://scrapytest.org/INDEX.html'}
)
headers={'Referer': 'http://scrapytest.org/INDEX.html'})
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)

76
tests/test_exporters.py
View File

@ -342,20 +342,22 @@ class XmlItemExporterTest(BaseItemExporterTest):
i2 = dict(name=u'bar', age=i1)
i3 = TestItem(name=u'buz', age=i2)
self.assertExportResult(i3,
b'<?xml version="1.0" encoding="utf-8"?>\n'
b'<items>'
b'<item>'
b'<age>'
b'<age>'
b'<age>22</age>'
b'<name>foo\xc2\xa3hoo</name>'
b'</age>'
b'<name>bar</name>'
b'</age>'
b'<name>buz</name>'
b'</item>'
b'</items>'
self.assertExportResult(
i3,
b"""<?xml version="1.0" encoding="utf-8"?>\n
<items>
<item>
<age>
<age>
<age>22</age>
<name>foo\xc2\xa3hoo</name>
</age>
<name>bar</name>
</age>
<name>buz</name>
</item>
</items>
"""
)
def test_nested_list_item(self):
@ -363,31 +365,35 @@ class XmlItemExporterTest(BaseItemExporterTest):
i2 = dict(name=u'bar', v2={"egg": ["spam"]})
i3 = TestItem(name=u'buz', age=[i1, i2])
self.assertExportResult(i3,
b'<?xml version="1.0" encoding="utf-8"?>\n'
b'<items>'
b'<item>'
b'<age>'
b'<value><name>foo</name></value>'
b'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
b'</age>'
b'<name>buz</name>'
b'</item>'
b'</items>'
self.assertExportResult(
i3,
b"""<?xml version="1.0" encoding="utf-8"?>\n
<items>
<item>
<age>
<value><name>foo</name></value>
<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>
</age>
<name>buz</name>
</item>
</items>
"""
)
def test_nonstring_types_item(self):
item = self._get_nonstring_types_item()
self.assertExportResult(item,
b'<?xml version="1.0" encoding="utf-8"?>\n'
b'<items>'
b'<item>'
b'<float>3.14</float>'
b'<boolean>False</boolean>'
b'<number>22</number>'
b'<time>2015-01-01 01:01:01</time>'
b'</item>'
b'</items>'
self.assertExportResult(
item,
b"""<?xml version="1.0" encoding="utf-8"?>\n
<items>
<item>
<float>3.14</float>
<boolean>False</boolean>
<number>22</number>
<time>2015-01-01 01:01:01</time>
</item>
</items>
"""
)

34
tests/test_linkextractors.py
View File

@ -413,24 +413,30 @@ class Base:
response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
lx = self.extractor_cls()
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
)
self.assertEqual(
lx.extract_links(response),
[
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
]
)
response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
lx = self.extractor_cls()
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
)
self.assertEqual(
lx.extract_links(response),
[
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
]
)
def test_link_wrong_href(self):
html = b"""

51
tests/test_utils_sitemap.py
View File

@ -58,10 +58,13 @@ class SitemapTest(unittest.TestCase):
</url>
</urlset>
""")
self.assertEqual(list(s),
[{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
{'loc': 'http://www.example.com/2', 'lastmod': ''},
])
self.assertEqual(
list(s),
[
{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
{'loc': 'http://www.example.com/2', 'lastmod': ''},
]
)
def test_sitemap_wrong_ns(self):
"""We have seen sitemaps with wrongs ns. Presumably, Google still works
@ -80,10 +83,13 @@ class SitemapTest(unittest.TestCase):
</url>
</urlset>
""")
self.assertEqual(list(s),
[{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
{'loc': 'http://www.example.com/2', 'lastmod': ''},
])
self.assertEqual(
list(s),
[
{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
{'loc': 'http://www.example.com/2', 'lastmod': ''},
]
)
def test_sitemap_wrong_ns2(self):
"""We have seen sitemaps with wrongs ns. Presumably, Google still works
@ -103,10 +109,13 @@ class SitemapTest(unittest.TestCase):
</urlset>
""")
assert s.type == 'urlset'
self.assertEqual(list(s),
[{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
{'loc': 'http://www.example.com/2', 'lastmod': ''},
])
self.assertEqual(
list(s),
[
{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
{'loc': 'http://www.example.com/2', 'lastmod': ''},
]
)
def test_sitemap_urls_from_robots(self):
robots = """User-agent: *
@ -195,11 +204,19 @@ Disallow: /forum/active/
</url>
</urlset>""")
self.assertEqual(list(s), [
{'loc': 'http://www.example.com/english/',
'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
}
])
self.assertEqual(
list(s),
[
{
'loc': 'http://www.example.com/english/',
'alternate': [
'http://www.example.com/deutsch/',
'http://www.example.com/schweiz-deutsch/',
'http://www.example.com/english/',
],
}
]
)
def test_xml_entity_expansion(self):
s = Sitemap(b"""<?xml version="1.0" encoding="utf-8"?>

40
tests/test_webclient.py
View File

@ -53,29 +53,29 @@ class ParseUrlTestCase(unittest.TestCase):
def testParse(self):
lip = '127.0.0.1'
tests = (
("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
("http://127.0.0.1", ('http', lip, lip, 80, '/')),
("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
("http://127.0.0.1", ('http', lip, lip, 80, '/')),
("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
)
("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
)
for url, test in tests:
test = tuple(