From c0efb271a23218656b5f629fde6d23c115a0c3de Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 27 Jan 2023 23:19:44 +0400 Subject: [PATCH] Ignore typing for twisted.internet.reactor globally. --- scrapy/core/downloader/handlers/http2.py | 8 ++++---- scrapy/utils/benchserver.py | 6 +++--- scrapy/utils/defer.py | 4 ++-- scrapy/utils/testsite.py | 4 ++-- setup.cfg | 3 +++ tests/CrawlerRunner/ip_address.py | 6 +++--- tests/mockserver.py | 10 +++++----- tests/test_engine.py | 2 +- 8 files changed, 23 insertions(+), 20 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 20cd50c5a..25ac0307b 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -28,7 +28,7 @@ class H2DownloadHandler: from twisted.internet import reactor - self._pool = H2ConnectionPool(reactor, settings) # type: ignore[arg-type] + self._pool = H2ConnectionPool(reactor, settings) self._context_factory = load_context_factory_from_settings(settings, crawler) @classmethod @@ -82,7 +82,7 @@ class ScrapyH2Agent: "Tunneling via CONNECT method using HTTP/2.0 is not yet supported" ) return self._ProxyAgent( - reactor=reactor, # type: ignore[arg-type] + reactor=reactor, context_factory=self._context_factory, proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")), connect_timeout=timeout, @@ -91,7 +91,7 @@ class ScrapyH2Agent: ) return self._Agent( - reactor=reactor, # type: ignore[arg-type] + reactor=reactor, context_factory=self._context_factory, connect_timeout=timeout, bind_address=bind_address, @@ -108,7 +108,7 @@ class ScrapyH2Agent: d = agent.request(request, spider) d.addCallback(self._cb_latency, request, start_time) - timeout_cl = reactor.callLater(timeout, d.cancel) # type: ignore[attr-defined] + timeout_cl = reactor.callLater(timeout, d.cancel) d.addBoth(self._cb_timeout, request, timeout, timeout_cl) return d diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 750d3c093..32bc2e38c 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -37,11 +37,11 @@ if __name__ == "__main__": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(8998, Site(root)) # type: ignore[attr-defined] + httpPort = reactor.listenTCP(8998, Site(root)) def _print_listening(): httpHost = httpPort.getHost() print(f"Bench server at http://{httpHost.host}:{httpHost.port}") - reactor.callWhenRunning(_print_listening) # type: ignore[attr-defined] - reactor.run() # type: ignore[attr-defined] + reactor.callWhenRunning(_print_listening) + reactor.run() diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 8fee5a7ed..7f2211877 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -40,7 +40,7 @@ def defer_fail(_failure: Failure) -> Deferred: from twisted.internet import reactor d: Deferred = Deferred() - reactor.callLater(0.1, d.errback, _failure) # type: ignore[attr-defined] + reactor.callLater(0.1, d.errback, _failure) return d @@ -54,7 +54,7 @@ def defer_succeed(result) -> Deferred: from twisted.internet import reactor d: Deferred = Deferred() - reactor.callLater(0.1, d.callback, result) # type: ignore[attr-defined] + reactor.callLater(0.1, d.callback, result) return d diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 119be1dfb..a47756c4b 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -50,6 +50,6 @@ def test_site(): if __name__ == "__main__": from twisted.internet import reactor - port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") # type: ignore[attr-defined] + port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") print(f"http://localhost:{port.getHost().port}/") - reactor.run() # type: ignore[attr-defined] + reactor.run() diff --git a/setup.cfg b/setup.cfg index af9c87945..db79c5821 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,6 +11,9 @@ ignore_missing_imports = true [mypy-twisted.internet.interfaces] follow_imports = skip +[mypy-twisted.internet.reactor] +follow_imports = skip + # FIXME: remove the following sections once the issues are solved [mypy-scrapy.downloadermiddlewares.httpproxy] diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index b9a4485a9..26db16dd6 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -41,10 +41,10 @@ if __name__ == "__main__": url = f"http://not.a.real.domain:{port}/echo" servers = [(mock_dns_server.host, mock_dns_server.port)] - reactor.installResolver(createResolver(servers=servers)) # type: ignore[attr-defined] + reactor.installResolver(createResolver(servers=servers)) configure_logging() runner = CrawlerRunner() d = runner.crawl(LocalhostSpider, url=url) - d.addBoth(lambda _: reactor.stop()) # type: ignore[attr-defined] - reactor.run() # type: ignore[attr-defined] + d.addBoth(lambda _: reactor.stop()) + reactor.run() diff --git a/tests/mockserver.py b/tests/mockserver.py index 185897373..eb1c03db7 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -375,9 +375,9 @@ if __name__ == "__main__": if args.type == "http": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(0, factory) # type: ignore[attr-defined] + httpPort = reactor.listenTCP(0, factory) contextFactory = ssl_context_factory() - httpsPort = reactor.listenSSL(0, factory, contextFactory) # type: ignore[attr-defined] + httpsPort = reactor.listenSSL(0, factory, contextFactory) def print_listening(): httpHost = httpPort.getHost() @@ -391,11 +391,11 @@ if __name__ == "__main__": clients = [MockDNSResolver()] factory = DNSServerFactory(clients=clients) protocol = dns.DNSDatagramProtocol(controller=factory) - listener = reactor.listenUDP(0, protocol) # type: ignore[attr-defined] + listener = reactor.listenUDP(0, protocol) def print_listening(): host = listener.getHost() print(f"{host.host}:{host.port}") - reactor.callWhenRunning(print_listening) # type: ignore[attr-defined] - reactor.run() # type: ignore[attr-defined] + reactor.callWhenRunning(print_listening) + reactor.run() diff --git a/tests/test_engine.py b/tests/test_engine.py index 5fb87424b..7ddb420ba 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -557,4 +557,4 @@ class EngineTest(unittest.TestCase): if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "runserver": start_test_site(debug=True) - reactor.run() # type: ignore[attr-defined] + reactor.run()