diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index 70c9003e5..734614aa5 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -2,7 +2,7 @@ from collections import deque from time import time -from twisted.internet import reactor +from twisted.internet import reactor # noqa: TID253 from twisted.web.resource import Resource from twisted.web.server import NOT_DONE_YET, Site diff --git a/pyproject.toml b/pyproject.toml index 68c1e07bb..871da8020 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -299,6 +299,8 @@ extend-select = [ "T10", # flake8-type-checking "TC", + # flake8-tidy-imports + "TID", # pyupgrade "UP", # pycodestyle warnings @@ -398,6 +400,11 @@ ignore = [ "SIM112", ] +[tool.ruff.lint.flake8-tidy-imports] +banned-module-level-imports = [ + "twisted.internet.reactor", +] + [tool.ruff.lint.per-file-ignores] # Circular import workarounds "scrapy/linkextractors/__init__.py" = ["E402"] diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 8f59c035c..cbe6427ea 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,4 +1,4 @@ -from twisted.internet import reactor # noqa: F401 +from twisted.internet import reactor # noqa: F401,TID253 from twisted.python import log import scrapy diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index 9901dd634..f7802fbc6 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -1,4 +1,4 @@ -from twisted.internet import reactor # noqa: F401 +from twisted.internet import reactor # noqa: F401,TID253 from twisted.python import log import scrapy diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py index 6c0102241..c275e0583 100644 --- a/tests/CrawlerRunner/change_reactor.py +++ b/tests/CrawlerRunner/change_reactor.py @@ -26,7 +26,7 @@ runner = CrawlerRunner() d = runner.crawl(NoRequestsSpider) -from twisted.internet import reactor # noqa: E402 +from twisted.internet import reactor # noqa: E402,TID253 d.addBoth(callback=lambda _: reactor.stop()) reactor.run() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 5e2184afb..207fc86ad 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -6,7 +6,6 @@ install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") from urllib.parse import urlparse -from twisted.internet import reactor from twisted.names import cache, resolve from twisted.names import hosts as hostsModule from twisted.names.client import Resolver @@ -44,6 +43,8 @@ class LocalhostSpider(Spider): if __name__ == "__main__": + from twisted.internet import reactor + with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: port = urlparse(mock_http_server.http_address).port url = f"http://not.a.real.domain:{port}/echo" diff --git a/tests/mockserver.py b/tests/mockserver.py index f5c12787a..e0ac127f2 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING from urllib.parse import urlencode from OpenSSL import SSL -from twisted.internet import defer, reactor, ssl +from twisted.internet import defer, ssl from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory @@ -114,6 +114,8 @@ class LeafResource(resource.Resource): isLeaf = True def deferRequest(self, request, delay, f, *a, **kw): + from twisted.internet import reactor + def _cancelrequest(_): # silence CancelledError d.addErrback(lambda _: None) @@ -378,6 +380,8 @@ def ssl_context_factory( if __name__ == "__main__": + from twisted.internet import reactor + parser = argparse.ArgumentParser() parser.add_argument( "-t", "--type", type=str, choices=("http", "dns"), default="http" diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index 5006e3689..c6fdb13ea 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -2,7 +2,7 @@ import sys from pathlib import Path from subprocess import PIPE, Popen -from .. import TWISTED_KEEPS_TRACEBACKS +from tests import TWISTED_KEEPS_TRACEBACKS class TestCmdlineCrawlPipeline: diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 1bffd69ed..ef77f7843 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -8,7 +8,6 @@ from typing import Any import OpenSSL.SSL import pytest -from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest @@ -40,6 +39,8 @@ class TestContextFactoryBase(unittest.TestCase): context_factory = None def _listen(self, site): + from twisted.internet import reactor + return reactor.listenSSL( 0, site, @@ -71,6 +72,8 @@ class TestContextFactoryBase(unittest.TestCase): client_context_factory: BrowserLikePolicyForHTTPS, body: str | None = None, ) -> bytes: + from twisted.internet import reactor + agent = Agent(reactor, contextFactory=client_context_factory) body_producer = _RequestBodyProducer(body.encode()) if body else None response: TxResponse = await maybe_deferred_to_future( diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 46322a747..159f403d0 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -8,7 +8,7 @@ from unittest import mock import pytest from testfixtures import LogCapture -from twisted.internet import defer, error, reactor +from twisted.internet import defer, error from twisted.web import server from twisted.web.error import SchemeNotSupported from twisted.web.http import H2_ENABLED @@ -59,6 +59,8 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): @defer.inlineCallbacks def test_download_with_maxsize_very_large_file(self): + from twisted.internet import reactor + with mock.patch("scrapy.core.http2.stream.logger") as logger: request = Request(self.getURL("largechunkedfile")) @@ -207,6 +209,8 @@ class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase): expected_http_proxy_request_body = b"/" def setUp(self): + from twisted.internet import reactor + site = server.Site(UriResource(), timeout=None) self.port = reactor.listenSSL( 0, diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index fc6ac5aee..dacadb075 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -12,7 +12,6 @@ from unittest import mock import pytest from twisted.cred import checkers, credentials, portal -from twisted.internet import reactor from twisted.protocols.ftp import FTPFactory, FTPRealm from twisted.trial import unittest from w3lib.url import path_to_file_uri @@ -310,6 +309,8 @@ class TestFTPBase(unittest.TestCase): ) def setUp(self): + from twisted.internet import reactor + # setup dirs and test file self.directory = Path(mkdtemp()) userdir = self.directory / self.username @@ -451,6 +452,8 @@ class TestAnonymousFTP(TestFTPBase): req_meta = {} def setUp(self): + from twisted.internet import reactor + # setup dir and test file self.directory = Path(mkdtemp()) for filename, content in self.test_files: diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 46e5972f7..5eaf66966 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -12,7 +12,7 @@ from unittest import mock import pytest from testfixtures import LogCapture -from twisted.internet import defer, error, reactor +from twisted.internet import defer, error from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util @@ -52,6 +52,8 @@ class ContentLengthHeaderResource(resource.Resource): class ChunkedResource(resource.Resource): def render(self, request): + from twisted.internet import reactor + def response(): request.write(b"chunked ") request.write(b"content\n") @@ -63,6 +65,8 @@ class ChunkedResource(resource.Resource): class BrokenChunkedResource(resource.Resource): def render(self, request): + from twisted.internet import reactor + def response(): request.write(b"chunked ") request.write(b"content\n") @@ -76,6 +80,8 @@ class BrokenChunkedResource(resource.Resource): class BrokenDownloadResource(resource.Resource): def render(self, request): + from twisted.internet import reactor + def response(): request.setHeader(b"Content-Length", b"20") request.write(b"partial") @@ -105,6 +111,8 @@ class EmptyContentTypeHeaderResource(resource.Resource): class LargeChunkedFileResource(resource.Resource): def render(self, request): + from twisted.internet import reactor + def response(): for i in range(1024): request.write(b"x" * 1024) @@ -133,6 +141,8 @@ class TestHttpBase(unittest.TestCase, ABC): raise NotImplementedError def setUp(self): + from twisted.internet import reactor + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) @@ -365,6 +375,9 @@ class TestHttp11Base(TestHttpBase): @defer.inlineCallbacks def test_download_with_maxsize_very_large_file(self): + from twisted.internet import reactor + + # TODO: the logger check is specific to scrapy.core.downloader.handlers.http11 with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger: request = Request(self.getURL("largechunkedfile")) @@ -501,6 +514,8 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC): raise NotImplementedError def setUp(self): + from twisted.internet import reactor + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) @@ -639,6 +654,8 @@ class TestHttpProxyBase(unittest.TestCase, ABC): raise NotImplementedError def setUp(self): + from twisted.internet import reactor + site = server.Site(UriResource(), timeout=None) wrapper = WrappingFactory(site) self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1") diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 9518f1835..04800896c 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING from unittest import mock import pytest -from twisted.internet import error, reactor +from twisted.internet import error from twisted.internet.defer import Deferred, maybeDeferred from twisted.python import failure from twisted.trial import unittest @@ -53,6 +53,8 @@ Disallow: /some/randome/page.html response = TextResponse("http://site.local/robots.txt", body=ROBOTS) def return_response(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -102,6 +104,8 @@ Disallow: /some/randome/page.html ) def return_response(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -124,6 +128,8 @@ Disallow: /some/randome/page.html response = Response("http://site.local/robots.txt") def return_response(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -145,6 +151,8 @@ Disallow: /some/randome/page.html err = error.DNSLookupError("Robotstxt address not found") def return_failure(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(err)) return deferred @@ -178,6 +186,8 @@ Disallow: /some/randome/page.html self.crawler.settings.set("ROBOTSTXT_OBEY", True) def ignore_request(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest())) return deferred diff --git a/tests/test_engine.py b/tests/test_engine.py index b60b510b2..1f79a081d 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -24,7 +24,7 @@ from urllib.parse import urlparse import attr from itemadapter import ItemAdapter from pydispatch import dispatcher -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.trial import unittest from twisted.web import server, static, util @@ -130,6 +130,8 @@ class ChangeCloseReasonSpider(MySpider): def start_test_site(debug=False): + from twisted.internet import reactor + root_dir = Path(tests_datadir, "test_site") r = static.File(str(root_dir)) r.putChild(b"redirect", util.Redirect(b"/redirected")) @@ -514,6 +516,8 @@ def test_request_scheduled_signal(caplog): if __name__ == "__main__": + from twisted.internet import reactor # pylint: disable=ungrouped-imports + if len(sys.argv) > 1 and sys.argv[1] == "runserver": start_test_site(debug=True) reactor.run() diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 7c1b38877..0605c2438 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -13,7 +13,6 @@ from unittest import mock, skipIf from urllib.parse import urlencode import pytest -from twisted.internet import reactor from twisted.internet.defer import ( CancelledError, Deferred, @@ -209,6 +208,8 @@ class TestHttps2ClientProtocol(TestCase): @inlineCallbacks def setUp(self): + from twisted.internet import reactor + # Initialize resource tree root = self._init_resource() self.site = Site(root, timeout=None) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d915fc2a3..2d0db6e25 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -4,7 +4,6 @@ import warnings import pytest from testfixtures import LogCapture -from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from twisted.trial import unittest @@ -329,6 +328,8 @@ class TestMediaPipeline(TestBaseMediaPipeline): rsp1 = Response("http://url") def rsp1_func(): + from twisted.internet import reactor + dfd = Deferred().addCallback(_check_downloading) reactor.callLater(0.1, dfd.callback, rsp1) return dfd diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 6dff321da..9b3ce6b0b 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -3,7 +3,7 @@ import asyncio import pytest from pydispatch import dispatcher from testfixtures import LogCapture -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.python.failure import Failure from twisted.trial import unittest @@ -65,6 +65,8 @@ class SendCatchLogDeferredTest(TestSendCatchLog): class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): def ok_handler(self, arg, handlers_called): + from twisted.internet import reactor + handlers_called.add(self.ok_handler) assert arg == "test" d = defer.Deferred() @@ -97,6 +99,8 @@ class SendCatchLogAsyncTest(TestSendCatchLog): class SendCatchLogAsyncTest2(SendCatchLogAsyncTest): def ok_handler(self, arg, handlers_called): + from twisted.internet import reactor + handlers_called.add(self.ok_handler) assert arg == "test" d = defer.Deferred() diff --git a/tests/test_webclient.py b/tests/test_webclient.py index c3c03d6c3..e580d51ca 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -12,7 +12,7 @@ from urllib.parse import urlparse import OpenSSL.SSL import pytest -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.internet.defer import inlineCallbacks from twisted.internet.testing import StringTransport from twisted.protocols.policies import WrappingFactory @@ -205,6 +205,8 @@ class EncodingResource(resource.Resource): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestWebClient(unittest.TestCase): def _listen(self, site): + from twisted.internet import reactor + return reactor.listenTCP(0, site, interface="127.0.0.1") def setUp(self): @@ -318,6 +320,8 @@ class TestWebClient(unittest.TestCase): assert b"404 - No Such Resource" in pageData def testFactoryInfo(self): + from twisted.internet import reactor + url = self.getURL("file") parsed = urlparse(url) factory = client.ScrapyHTTPClientFactory(Request(url))