diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index ad5c71fbf..ca5bc9a5c 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -49,6 +49,7 @@ You can usually fix the issue by moving those offending module-level Twisted
imports to the method or function definitions where they are used. For example,
if you have something like:
+.. skip: next
.. code-block:: python
from twisted.internet import reactor
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index db65fb993..4e9519eef 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -2003,6 +2003,7 @@ reactor is installed.
In order to use the reactor installed by Scrapy:
+.. skip: next
.. code-block:: python
import scrapy
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index e7e7440a8..70674c021 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -30,6 +30,7 @@ from scrapy.crawler import (
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.throttle import AutoThrottle
from scrapy.settings import Settings, default_settings
+from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_coro
from scrapy.utils.log import configure_logging, get_scrapy_root_handler
from scrapy.utils.spider import DefaultSpider
@@ -932,8 +933,6 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
@inlineCallbacks
def test_shutdown_forced(self):
- from twisted.internet import reactor
-
sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
args = self.get_script_args("sleeping.py", "10")
p = PopenSpawn(args, timeout=5)
@@ -943,7 +942,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
p.expect_exact("shutting down gracefully")
# sending the second signal too fast often causes problems
d = Deferred()
- reactor.callLater(0.01, d.callback, None)
+ call_later(0.01, d.callback, None)
yield d
p.kill(sig)
p.expect_exact("forcing unclean shutdown")
diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py
new file mode 100644
index 000000000..a43fe318d
--- /dev/null
+++ b/tests/test_downloader_handler_twisted_ftp.py
@@ -0,0 +1,194 @@
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+from tempfile import mkstemp
+from typing import TYPE_CHECKING, Any
+
+import pytest
+from pytest_twisted import async_yield_fixture
+from twisted.cred import checkers, credentials, portal
+
+from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
+from scrapy.http import HtmlResponse, Request, Response
+from scrapy.http.response.text import TextResponse
+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.python import to_bytes
+from scrapy.utils.spider import DefaultSpider
+from scrapy.utils.test import get_crawler
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator, Generator
+
+
+class TestFTPBase:
+ username = "scrapy"
+ password = "passwd"
+ req_meta: dict[str, Any] = {"ftp_user": username, "ftp_password": password}
+
+ test_files = (
+ ("file.txt", b"I have the power!"),
+ ("file with spaces.txt", b"Moooooooooo power!"),
+ ("html-file-without-extension", b"\n."),
+ )
+
+ def _create_files(self, root: Path) -> None:
+ userdir = root / self.username
+ userdir.mkdir()
+ for filename, content in self.test_files:
+ (userdir / filename).write_bytes(content)
+
+ def _get_factory(self, root):
+ from twisted.protocols.ftp import FTPFactory, FTPRealm
+
+ realm = FTPRealm(anonymousRoot=str(root), userHome=str(root))
+ p = portal.Portal(realm)
+ users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
+ users_checker.addUser(self.username, self.password)
+ p.registerChecker(users_checker, credentials.IUsernamePassword)
+ return FTPFactory(portal=p)
+
+ @async_yield_fixture
+ async def server_url(self, tmp_path: Path) -> AsyncGenerator[str]:
+ from twisted.internet import reactor
+
+ self._create_files(tmp_path)
+ factory = self._get_factory(tmp_path)
+ port = reactor.listenTCP(0, factory, interface="127.0.0.1")
+ portno = port.getHost().port
+
+ yield f"https://127.0.0.1:{portno}/"
+
+ await port.stopListening()
+
+ @staticmethod
+ @pytest.fixture
+ def dh() -> Generator[FTPDownloadHandler]:
+ crawler = get_crawler()
+ dh = build_from_crawler(FTPDownloadHandler, crawler)
+
+ yield dh
+
+ # if the test was skipped, there will be no client attribute
+ if hasattr(dh, "client"):
+ assert dh.client.transport
+ dh.client.transport.loseConnection()
+
+ @staticmethod
+ async def download_request(dh: FTPDownloadHandler, request: Request) -> Response:
+ return await maybe_deferred_to_future(
+ dh.download_request(request, DefaultSpider())
+ )
+
+ @deferred_f_from_coro_f
+ async def test_ftp_download_success(
+ self, server_url: str, dh: FTPDownloadHandler
+ ) -> None:
+ request = Request(url=server_url + "file.txt", meta=self.req_meta)
+ r = await self.download_request(dh, request)
+ assert r.status == 200
+ assert r.body == b"I have the power!"
+ assert r.headers == {b"Local Filename": [b""], b"Size": [b"17"]}
+ assert r.protocol is None
+
+ @deferred_f_from_coro_f
+ async def test_ftp_download_path_with_spaces(
+ self, server_url: str, dh: FTPDownloadHandler
+ ) -> None:
+ request = Request(
+ url=server_url + "file with spaces.txt",
+ meta=self.req_meta,
+ )
+ r = await self.download_request(dh, request)
+ assert r.status == 200
+ assert r.body == b"Moooooooooo power!"
+ assert r.headers == {b"Local Filename": [b""], b"Size": [b"18"]}
+
+ @deferred_f_from_coro_f
+ async def test_ftp_download_nonexistent(
+ self, server_url: str, dh: FTPDownloadHandler
+ ) -> None:
+ request = Request(url=server_url + "nonexistent.txt", meta=self.req_meta)
+ r = await self.download_request(dh, request)
+ assert r.status == 404
+
+ @deferred_f_from_coro_f
+ async def test_ftp_local_filename(
+ self, server_url: str, dh: FTPDownloadHandler
+ ) -> None:
+ f, local_fname = mkstemp()
+ fname_bytes = to_bytes(local_fname)
+ local_path = Path(local_fname)
+ os.close(f)
+ meta = {"ftp_local_filename": fname_bytes}
+ meta.update(self.req_meta)
+ request = Request(url=server_url + "file.txt", meta=meta)
+ r = await self.download_request(dh, request)
+ assert r.body == fname_bytes
+ assert r.headers == {b"Local Filename": [fname_bytes], b"Size": [b"17"]}
+ assert local_path.exists()
+ assert local_path.read_bytes() == b"I have the power!"
+ local_path.unlink()
+
+ @pytest.mark.parametrize(
+ ("filename", "response_class"),
+ [
+ ("file.txt", TextResponse),
+ ("html-file-without-extension", HtmlResponse),
+ ],
+ )
+ @deferred_f_from_coro_f
+ async def test_response_class(
+ self,
+ filename: str,
+ response_class: type[Response],
+ server_url: str,
+ dh: FTPDownloadHandler,
+ ) -> None:
+ f, local_fname = mkstemp()
+ local_fname_path = Path(local_fname)
+ os.close(f)
+ meta = {}
+ meta.update(self.req_meta)
+ request = Request(url=server_url + filename, meta=meta)
+ r = await self.download_request(dh, request)
+ assert type(r) is response_class # pylint: disable=unidiomatic-typecheck
+ local_fname_path.unlink()
+
+
+class TestFTP(TestFTPBase):
+ @deferred_f_from_coro_f
+ async def test_invalid_credentials(
+ self, server_url: str, dh: FTPDownloadHandler, reactor_pytest: str
+ ) -> None:
+ if reactor_pytest == "asyncio" and sys.platform == "win32":
+ pytest.skip(
+ "This test produces DirtyReactorAggregateError on Windows with asyncio"
+ )
+
+ from twisted.protocols.ftp import ConnectionLost
+
+ meta = dict(self.req_meta)
+ meta.update({"ftp_password": "invalid"})
+ request = Request(url=server_url + "file.txt", meta=meta)
+ with pytest.raises(ConnectionLost):
+ await self.download_request(dh, request)
+
+
+class TestAnonymousFTP(TestFTPBase):
+ username = "anonymous"
+ req_meta = {}
+
+ def _create_files(self, root: Path) -> None:
+ for filename, content in self.test_files:
+ (root / filename).write_bytes(content)
+
+ def _get_factory(self, tmp_path):
+ from twisted.protocols.ftp import FTPFactory, FTPRealm
+
+ realm = FTPRealm(anonymousRoot=str(tmp_path))
+ p = portal.Portal(realm)
+ p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous)
+ return FTPFactory(portal=p, userAnonymous=self.username)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 9aa53edd9..e37932fbb 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -4,35 +4,25 @@ from __future__ import annotations
import contextlib
import os
-import sys
from pathlib import Path
from tempfile import mkdtemp, mkstemp
-from typing import TYPE_CHECKING, Any
from unittest import mock
import pytest
-from pytest_twisted import async_yield_fixture
-from twisted.cred import checkers, credentials, portal
from w3lib.url import path_to_file_uri
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
from scrapy.core.downloader.handlers.file import FileDownloadHandler
-from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
from scrapy.exceptions import NotConfigured
-from scrapy.http import HtmlResponse, Request, Response
-from scrapy.http.response.text import TextResponse
+from scrapy.http import Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
-from scrapy.utils.python import to_bytes
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
-if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Generator
-
class DummyDH:
lazy = False
@@ -305,177 +295,6 @@ class TestS3:
)
-class TestFTPBase:
- username = "scrapy"
- password = "passwd"
- req_meta: dict[str, Any] = {"ftp_user": username, "ftp_password": password}
-
- test_files = (
- ("file.txt", b"I have the power!"),
- ("file with spaces.txt", b"Moooooooooo power!"),
- ("html-file-without-extension", b"\n."),
- )
-
- def _create_files(self, root: Path) -> None:
- userdir = root / self.username
- userdir.mkdir()
- for filename, content in self.test_files:
- (userdir / filename).write_bytes(content)
-
- def _get_factory(self, root):
- from twisted.protocols.ftp import FTPFactory, FTPRealm
-
- realm = FTPRealm(anonymousRoot=str(root), userHome=str(root))
- p = portal.Portal(realm)
- users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
- users_checker.addUser(self.username, self.password)
- p.registerChecker(users_checker, credentials.IUsernamePassword)
- return FTPFactory(portal=p)
-
- @async_yield_fixture
- async def server_url(self, tmp_path: Path) -> AsyncGenerator[str]:
- from twisted.internet import reactor
-
- self._create_files(tmp_path)
- factory = self._get_factory(tmp_path)
- port = reactor.listenTCP(0, factory, interface="127.0.0.1")
- portno = port.getHost().port
-
- yield f"https://127.0.0.1:{portno}/"
-
- await port.stopListening()
-
- @staticmethod
- @pytest.fixture
- def dh() -> Generator[FTPDownloadHandler]:
- crawler = get_crawler()
- dh = build_from_crawler(FTPDownloadHandler, crawler)
-
- yield dh
-
- # if the test was skipped, there will be no client attribute
- if hasattr(dh, "client"):
- assert dh.client.transport
- dh.client.transport.loseConnection()
-
- @staticmethod
- async def download_request(dh: FTPDownloadHandler, request: Request) -> Response:
- return await maybe_deferred_to_future(
- dh.download_request(request, DefaultSpider())
- )
-
- @deferred_f_from_coro_f
- async def test_ftp_download_success(
- self, server_url: str, dh: FTPDownloadHandler
- ) -> None:
- request = Request(url=server_url + "file.txt", meta=self.req_meta)
- r = await self.download_request(dh, request)
- assert r.status == 200
- assert r.body == b"I have the power!"
- assert r.headers == {b"Local Filename": [b""], b"Size": [b"17"]}
- assert r.protocol is None
-
- @deferred_f_from_coro_f
- async def test_ftp_download_path_with_spaces(
- self, server_url: str, dh: FTPDownloadHandler
- ) -> None:
- request = Request(
- url=server_url + "file with spaces.txt",
- meta=self.req_meta,
- )
- r = await self.download_request(dh, request)
- assert r.status == 200
- assert r.body == b"Moooooooooo power!"
- assert r.headers == {b"Local Filename": [b""], b"Size": [b"18"]}
-
- @deferred_f_from_coro_f
- async def test_ftp_download_nonexistent(
- self, server_url: str, dh: FTPDownloadHandler
- ) -> None:
- request = Request(url=server_url + "nonexistent.txt", meta=self.req_meta)
- r = await self.download_request(dh, request)
- assert r.status == 404
-
- @deferred_f_from_coro_f
- async def test_ftp_local_filename(
- self, server_url: str, dh: FTPDownloadHandler
- ) -> None:
- f, local_fname = mkstemp()
- fname_bytes = to_bytes(local_fname)
- local_path = Path(local_fname)
- os.close(f)
- meta = {"ftp_local_filename": fname_bytes}
- meta.update(self.req_meta)
- request = Request(url=server_url + "file.txt", meta=meta)
- r = await self.download_request(dh, request)
- assert r.body == fname_bytes
- assert r.headers == {b"Local Filename": [fname_bytes], b"Size": [b"17"]}
- assert local_path.exists()
- assert local_path.read_bytes() == b"I have the power!"
- local_path.unlink()
-
- @pytest.mark.parametrize(
- ("filename", "response_class"),
- [
- ("file.txt", TextResponse),
- ("html-file-without-extension", HtmlResponse),
- ],
- )
- @deferred_f_from_coro_f
- async def test_response_class(
- self,
- filename: str,
- response_class: type[Response],
- server_url: str,
- dh: FTPDownloadHandler,
- ) -> None:
- f, local_fname = mkstemp()
- local_fname_path = Path(local_fname)
- os.close(f)
- meta = {}
- meta.update(self.req_meta)
- request = Request(url=server_url + filename, meta=meta)
- r = await self.download_request(dh, request)
- assert type(r) is response_class # pylint: disable=unidiomatic-typecheck
- local_fname_path.unlink()
-
-
-class TestFTP(TestFTPBase):
- @deferred_f_from_coro_f
- async def test_invalid_credentials(
- self, server_url: str, dh: FTPDownloadHandler, reactor_pytest: str
- ) -> None:
- if reactor_pytest == "asyncio" and sys.platform == "win32":
- pytest.skip(
- "This test produces DirtyReactorAggregateError on Windows with asyncio"
- )
-
- from twisted.protocols.ftp import ConnectionLost
-
- meta = dict(self.req_meta)
- meta.update({"ftp_password": "invalid"})
- request = Request(url=server_url + "file.txt", meta=meta)
- with pytest.raises(ConnectionLost):
- await self.download_request(dh, request)
-
-
-class TestAnonymousFTP(TestFTPBase):
- username = "anonymous"
- req_meta = {}
-
- def _create_files(self, root: Path) -> None:
- for filename, content in self.test_files:
- (root / filename).write_bytes(content)
-
- def _get_factory(self, tmp_path):
- from twisted.protocols.ftp import FTPFactory, FTPRealm
-
- realm = FTPRealm(anonymousRoot=str(tmp_path))
- p = portal.Portal(realm)
- p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous)
- return FTPFactory(portal=p, userAnonymous=self.username)
-
-
class TestDataURI:
def setup_method(self):
crawler = get_crawler()
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index bbaf9696b..3784d0933 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -17,6 +17,7 @@ from twisted.web.http import _DataLoss
from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse
from scrapy.spiders import Spider
+from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import (
deferred_f_from_coro_f,
deferred_from_coro,
@@ -305,8 +306,6 @@ class TestHttp11Base(TestHttpBase):
async def test_download_with_maxsize_very_large_file(
self, mockserver: MockServer, download_handler: DownloadHandlerProtocol
) -> None:
- from twisted.internet import reactor
-
# TODO: the logger check is specific to scrapy.core.downloader.handlers.http11
with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger:
request = Request(
@@ -326,7 +325,7 @@ class TestHttp11Base(TestHttpBase):
# after closing the connection.
d: defer.Deferred[mock.Mock] = defer.Deferred()
d.addCallback(check)
- reactor.callLater(0.1, d.callback, logger)
+ call_later(0.1, d.callback, logger)
await maybe_deferred_to_future(d)
@deferred_f_from_coro_f
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index 2bb5d2ff4..6915e1463 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -138,7 +138,6 @@ class TestRequestSendOrder:
def get_num(self, request_or_response: Request | Response):
return int(request_or_response.url.rsplit("&", maxsplit=1)[1])
- @deferred_f_from_coro_f
async def _test_request_order(
self,
start_nums,
@@ -218,14 +217,12 @@ class TestRequestSendOrder:
return
yield
- await maybe_deferred_to_future(
- self._test_request_order(
- start_nums=nums,
- settings={"CONCURRENT_REQUESTS": 1},
- response_seconds=response_seconds,
- start_fn=start,
- parse_fn=parse,
- )
+ await self._test_request_order(
+ start_nums=nums,
+ settings={"CONCURRENT_REQUESTS": 1},
+ response_seconds=response_seconds,
+ start_fn=start,
+ parse_fn=parse,
)
@deferred_f_from_coro_f
@@ -260,17 +257,15 @@ class TestRequestSendOrder:
return
yield
- await maybe_deferred_to_future(
- self._test_request_order(
- start_nums=nums,
- settings={
- "CONCURRENT_REQUESTS": 1,
- "SCHEDULER_START_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue",
- },
- response_seconds=response_seconds,
- start_fn=start,
- parse_fn=parse,
- )
+ await self._test_request_order(
+ start_nums=nums,
+ settings={
+ "CONCURRENT_REQUESTS": 1,
+ "SCHEDULER_START_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue",
+ },
+ response_seconds=response_seconds,
+ start_fn=start,
+ parse_fn=parse,
)
@deferred_f_from_coro_f
@@ -320,17 +315,15 @@ class TestRequestSendOrder:
return
yield
- await maybe_deferred_to_future(
- self._test_request_order(
- start_nums=nums,
- settings={
- "CONCURRENT_REQUESTS": 1,
- "SCHEDULER_START_MEMORY_QUEUE": None,
- },
- response_seconds=response_seconds,
- start_fn=start,
- parse_fn=parse,
- )
+ await self._test_request_order(
+ start_nums=nums,
+ settings={
+ "CONCURRENT_REQUESTS": 1,
+ "SCHEDULER_START_MEMORY_QUEUE": None,
+ },
+ response_seconds=response_seconds,
+ start_fn=start,
+ parse_fn=parse,
)
# Examples from the “Start requests” section of the documentation about
@@ -350,14 +343,12 @@ class TestRequestSendOrder:
request = self.request(num, response_seconds, download_slots)
yield request
- await maybe_deferred_to_future(
- self._test_request_order(
- start_nums=start_nums,
- cb_nums=cb_nums,
- settings={
- "CONCURRENT_REQUESTS": 1,
- },
- response_seconds=response_seconds,
- start_fn=start,
- )
+ await self._test_request_order(
+ start_nums=start_nums,
+ cb_nums=cb_nums,
+ settings={
+ "CONCURRENT_REQUESTS": 1,
+ },
+ response_seconds=response_seconds,
+ start_fn=start,
)
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 3247da61b..d100d6b71 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -44,6 +44,11 @@ if TYPE_CHECKING:
from scrapy.core.http2.protocol import H2ClientProtocol
+pytestmark = pytest.mark.skipif(
+ not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"
+)
+
+
def generate_random_string(size: int) -> str:
return "".join(random.choices(string.ascii_uppercase + string.digits, k=size))
@@ -187,7 +192,6 @@ async def make_request(client: H2ClientProtocol, request: Request) -> Response:
return await maybe_deferred_to_future(make_request_dfd(client, request))
-@pytest.mark.skipif(not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled")
class TestHttps2ClientProtocol:
scheme = "https"
host = "localhost"
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index b309c5d8e..42ae897e8 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -14,6 +14,7 @@ from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FileException
from scrapy.pipelines.media import MediaPipeline
from scrapy.spiders import Spider
+from scrapy.utils.asyncio import call_later
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.signal import disconnect_all
from scrapy.utils.test import get_crawler
@@ -337,10 +338,8 @@ class TestMediaPipeline(TestBaseMediaPipeline):
rsp1 = Response("http://url")
def rsp1_func():
- from twisted.internet import reactor
-
dfd = Deferred().addCallback(_check_downloading)
- reactor.callLater(0.1, dfd.callback, rsp1)
+ call_later(0.1, dfd.callback, rsp1)
return dfd
def rsp2_func():
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 064033f01..cba6a713c 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -4,6 +4,7 @@ import pytest
from twisted.internet.defer import Deferred, inlineCallbacks
from scrapy import Request, Spider, signals
+from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.mockserver.http import MockServer
@@ -30,9 +31,7 @@ class DeferredPipeline:
class AsyncDefPipeline:
async def process_item(self, item, spider):
d = Deferred()
- from twisted.internet import reactor
-
- reactor.callLater(0, d.callback, None)
+ call_later(0, d.callback, None)
await maybe_deferred_to_future(d)
item["pipeline_passed"] = True
return item
@@ -41,9 +40,8 @@ class AsyncDefPipeline:
class AsyncDefAsyncioPipeline:
async def process_item(self, item, spider):
d = Deferred()
- from twisted.internet import reactor
-
- reactor.callLater(0, d.callback, None)
+ loop = asyncio.get_event_loop()
+ loop.call_later(0, d.callback, None)
await deferred_to_future(d)
await asyncio.sleep(0.2)
item["pipeline_passed"] = await get_from_asyncio_queue(True)
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index b0f69c1ff..bc2404d8b 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -14,6 +14,7 @@ from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.asyncgen import collect_asyncgen
+from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.test import get_crawler
@@ -220,9 +221,7 @@ class ProcessSpiderExceptionAsyncIteratorMiddleware:
async def process_spider_exception(self, response, exception, spider):
yield {"foo": 1}
d = defer.Deferred()
- from twisted.internet import reactor
-
- reactor.callLater(0, d.callback, None)
+ call_later(0, d.callback, None)
await maybe_deferred_to_future(d)
yield {"foo": 2}
yield {"foo": 3}
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index d90d3ed95..6e4bdb49a 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -7,6 +7,7 @@ from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.python.failure import Failure
+from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.signal import (
send_catch_log,
@@ -65,12 +66,10 @@ class TestSendCatchLogDeferred(TestSendCatchLog):
class TestSendCatchLogDeferred2(TestSendCatchLogDeferred):
def ok_handler(self, arg, handlers_called):
- from twisted.internet import reactor
-
handlers_called.add(self.ok_handler)
assert arg == "test"
d = defer.Deferred()
- reactor.callLater(0, d.callback, "OK")
+ call_later(0, d.callback, "OK")
return d
@@ -98,12 +97,10 @@ class TestSendCatchLogAsync(TestSendCatchLog):
class TestSendCatchLogAsync2(TestSendCatchLogAsync):
def ok_handler(self, arg, handlers_called):
- from twisted.internet import reactor
-
handlers_called.add(self.ok_handler)
assert arg == "test"
d = defer.Deferred()
- reactor.callLater(0, d.callback, "OK")
+ call_later(0, d.callback, "OK")
return d