diff --git a/pyproject.toml b/pyproject.toml index 9db636a08..e29393a04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -398,6 +398,10 @@ ignore = [ [tool.ruff.lint.flake8-tidy-imports] banned-module-level-imports = [ "twisted.internet.reactor", + # indirectly imports twisted.conch.insults.helper which imports twisted.internet.reactor + "twisted.conch.manhole", + # directly imports twisted.internet.reactor + "twisted.protocols.ftp", ] [tool.ruff.lint.isort] diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 0ad10baff..1d947b1e3 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -37,7 +37,6 @@ from typing import TYPE_CHECKING, Any, BinaryIO from urllib.parse import unquote from twisted.internet.protocol import ClientCreator, Protocol -from twisted.protocols.ftp import CommandFailed, FTPClient from scrapy.http import Response from scrapy.responsetypes import responsetypes @@ -46,6 +45,7 @@ from scrapy.utils.python import to_bytes if TYPE_CHECKING: from twisted.internet.defer import Deferred + from twisted.protocols.ftp import FTPClient from twisted.python.failure import Failure # typing.Self requires Python 3.11 @@ -101,6 +101,7 @@ class FTPDownloadHandler: def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: from twisted.internet import reactor + from twisted.protocols.ftp import FTPClient parsed_url = urlparse_cached(request) user = request.meta.get("ftp_user", self.default_user) @@ -138,6 +139,8 @@ class FTPDownloadHandler: return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type] def _failed(self, result: Failure, request: Request) -> Response: + from twisted.protocols.ftp import CommandFailed + message = result.getErrorMessage() if result.type == CommandFailed: m = _CODE_RE.search(message) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 9fe71817a..094a0195e 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -12,7 +12,7 @@ import os import pprint from typing import TYPE_CHECKING, Any -from twisted.conch import manhole, telnet +from twisted.conch import telnet from twisted.conch.insults import insults from twisted.internet import protocol @@ -88,6 +88,8 @@ class TelnetConsole(protocol.ServerFactory): ): raise ValueError("Invalid credentials") + from twisted.conch import manhole + protocol = telnet.TelnetBootstrapProtocol( insults.ServerProtocol, manhole.Manhole, self._get_telnet_vars() ) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index eba14c0c3..9aa53edd9 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -13,7 +13,6 @@ from unittest import mock import pytest from pytest_twisted import async_yield_fixture from twisted.cred import checkers, credentials, portal -from twisted.protocols.ftp import ConnectionLost, FTPFactory, FTPRealm from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers @@ -324,6 +323,8 @@ class TestFTPBase: (userdir / filename).write_bytes(content) def _get_factory(self, root): + from twisted.protocols.ftp import FTPFactory, FTPRealm + realm = FTPRealm(anonymousRoot=str(root), userHome=str(root)) p = portal.Portal(realm) users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() @@ -449,6 +450,8 @@ class TestFTP(TestFTPBase): "This test produces DirtyReactorAggregateError on Windows with asyncio" ) + from twisted.protocols.ftp import ConnectionLost + meta = dict(self.req_meta) meta.update({"ftp_password": "invalid"}) request = Request(url=server_url + "file.txt", meta=meta) @@ -465,6 +468,8 @@ class TestAnonymousFTP(TestFTPBase): (root / filename).write_bytes(content) def _get_factory(self, tmp_path): + from twisted.protocols.ftp import FTPFactory, FTPRealm + realm = FTPRealm(anonymousRoot=str(tmp_path)) p = portal.Portal(realm) p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous)