From 13c5ad7e688271ae47f2a489bf665cb15387b3a3 Mon Sep 17 00:00:00 2001 From: "Alexandr N. Zamaraev" Date: Tue, 16 Aug 2022 11:03:37 +0700 Subject: [PATCH 1/2] Partial in is_generator_with_return_value See path in #5592 --- scrapy/utils/misc.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 1221b39b2..2e25f6421 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -9,6 +9,7 @@ from collections import deque from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules +from functools import partial from w3lib.html import replace_entities @@ -226,7 +227,11 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - code = re.sub(r"^[\t ]+", "", inspect.getsource(callable)) + func = callable + while isinstance(func, partial): + func = func.func + + code = re.sub(r"^[\t ]+", "", inspect.getsource(func)) tree = ast.parse(code) for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): From b71d0292d5ff85d652e2943ee1a727a128b55594 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 27 Oct 2022 18:13:47 +0600 Subject: [PATCH 2/2] Add a test for processing partial callbacks. --- .../test_return_with_argument_inside_generator.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 72277d701..562f72fee 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,5 +1,6 @@ import unittest import warnings +from functools import partial from unittest import mock from scrapy.utils.misc import is_generator_with_return_value, warn_on_generator_with_return_value @@ -254,3 +255,10 @@ https://example.org warn_on_generator_with_return_value(None, top_level_return_none) self.assertEqual(len(w), 1) self.assertIn('Unable to determine', str(w[0].message)) + + def test_partial(self): + def cb(arg1, arg2): + yield {} + + partial_cb = partial(cb, arg1=42) + assert not is_generator_with_return_value(partial_cb)