Merge pull request #5668 from scrapy/update-tools

Update tool versions

.flake8

@@ -4,6 +4,8 @@ max-line-length = 119
 ignore = W503
 
 exclude =
+    docs/conf.py
+
 # Exclude files that are meant to provide top-level imports
 # E402: Module level import not at top of file
 # F401: Module imported but unused

conftest.py

@@ -42,16 +42,6 @@ def chdir(tmpdir):
     tmpdir.chdir()
 
 
-def pytest_collection_modifyitems(session, config, items):
-    # Avoid executing tests when executing `--flake8` flag (pytest-flake8)
-    try:
-        from pytest_flake8 import Flake8Item
-        if config.getoption('--flake8'):
-            items[:] = [item for item in items if isinstance(item, Flake8Item)]
-    except ImportError:
-        pass
-
-
 def pytest_addoption(parser):
     parser.addoption(
         "--reactor",

docs/_ext/scrapydocs.py

@@ -80,24 +80,24 @@ def replace_settingslist_nodes(app, doctree, fromdocname):
 
 def setup(app):
     app.add_crossref_type(
-        directivename = "setting",
-        rolename      = "setting",
-        indextemplate = "pair: %s; setting",
+        directivename="setting",
+        rolename="setting",
+        indextemplate="pair: %s; setting",
     )
     app.add_crossref_type(
-        directivename = "signal",
-        rolename      = "signal",
-        indextemplate = "pair: %s; signal",
+        directivename="signal",
+        rolename="signal",
+        indextemplate="pair: %s; signal",
     )
     app.add_crossref_type(
-        directivename = "command",
-        rolename      = "command",
-        indextemplate = "pair: %s; command",
+        directivename="command",
+        rolename="command",
+        indextemplate="pair: %s; command",
     )
     app.add_crossref_type(
-        directivename = "reqmeta",
-        rolename      = "reqmeta",
-        indextemplate = "pair: %s; reqmeta",
+        directivename="reqmeta",
+        rolename="reqmeta",
+        indextemplate="pair: %s; reqmeta",
     )
     app.add_role('source', source_role)
     app.add_role('commit', commit_role)

scrapy/utils/testsite.py

@@ -23,7 +23,7 @@ class NoMetaRefreshRedirect(util.Redirect):
     def render(self, request):
         content = util.Redirect.render(self, request)
         return content.replace(b'http-equiv=\"refresh\"',
-            b'http-no-equiv=\"do-not-refresh-me\"')
+                               b'http-no-equiv=\"do-not-refresh-me\"')
 
 
 def test_site():

tests/CrawlerProcess/asyncio_deferred_signal.py

@@ -5,7 +5,6 @@ from typing import Optional
 from scrapy import Spider
 from scrapy.crawler import CrawlerProcess
 from scrapy.utils.defer import deferred_from_coro
-from twisted.internet.defer import Deferred
 
 
 class UppercasePipeline:

tests/CrawlerProcess/asyncio_enabled_reactor.py

@@ -6,8 +6,8 @@ if sys.version_info >= (3, 8) and sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 asyncioreactor.install(asyncio.get_event_loop())
 
-import scrapy
-from scrapy.crawler import CrawlerProcess
+import scrapy  # noqa: E402
+from scrapy.crawler import CrawlerProcess  # noqa: E402
 
 
 class NoRequestsSpider(scrapy.Spider):

tests/CrawlerProcess/reactor_default.py

@@ -1,6 +1,6 @@
 import scrapy
 from scrapy.crawler import CrawlerProcess
-from twisted.internet import reactor
+from twisted.internet import reactor  # noqa: F401
 
 
 class NoRequestsSpider(scrapy.Spider):
@@ -14,4 +14,3 @@ process = CrawlerProcess(settings={})
 
 process.crawl(NoRequestsSpider)
 process.start()
-

tests/CrawlerProcess/reactor_default_twisted_reactor_select.py

@@ -1,6 +1,6 @@
 import scrapy
 from scrapy.crawler import CrawlerProcess
-from twisted.internet import reactor
+from twisted.internet import reactor  # noqa: F401
 
 
 class NoRequestsSpider(scrapy.Spider):
@@ -16,5 +16,3 @@ process = CrawlerProcess(settings={
 
 process.crawl(NoRequestsSpider)
 process.start()
-
-

tests/CrawlerProcess/reactor_select.py

@@ -15,5 +15,3 @@ process = CrawlerProcess(settings={})
 
 process.crawl(NoRequestsSpider)
 process.start()
-
-

tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py

@@ -25,7 +25,3 @@ process = CrawlerProcess(settings={
 
 process.crawl(NoRequestsSpider)
 process.start()
-
-
-
-

tests/CrawlerProcess/reactor_select_twisted_reactor_select.py

@@ -17,6 +17,3 @@ process = CrawlerProcess(settings={
 
 process.crawl(NoRequestsSpider)
 process.start()
-
-
-

tests/test_http_response.py

@@ -705,7 +705,8 @@ class HtmlResponseTest(TextResponseTest):
 
     def test_html_encoding(self):
 
-        body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+        body = b"""<html><head><title>Some page</title>
+        <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
         </head><body>Price: \xa3100</body></html>'
         """
         r1 = self.response_class("http://www.example.com", body=body)
@@ -719,7 +720,8 @@ class HtmlResponseTest(TextResponseTest):
         self._assert_response_values(r2, 'iso-8859-1', body)
 
         # for conflicting declarations headers must take precedence
-        body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+        body = b"""<html><head><title>Some page</title>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
         </head><body>Price: \xa3100</body></html>'
         """
         r3 = self.response_class("http://www.example.com", body=body,

tox.ini

@@ -38,7 +38,8 @@ install_command =
 basepython = python3
 deps =
     lxml-stubs==0.2.0
-    mypy==0.971
+    mypy==0.982
+    types-attrs==19.1.0
     types-pyOpenSSL==21.0.0
     types-setuptools==57.0.0
 commands =
@@ -57,17 +58,16 @@ deps =
     {[testenv]deps}
     # Twisted[http2] is required to import some files
     Twisted[http2]>=17.9.0
-    pytest-flake8==1.1.1
-    flake8==4.0.1
+    flake8==5.0.4
 commands =
-    pytest --flake8 {posargs:docs scrapy tests}
+    flake8 {posargs:docs scrapy tests}
 
 [testenv:pylint]
 # reppy does not support Python 3.9+
 basepython = python3.8
 deps =
     {[testenv:extra-deps]deps}
-    pylint==2.14.5
+    pylint==2.15.3
 commands =
     pylint conftest.py docs extras scrapy setup.py tests