From b61b71c6f015d45f6e98a4280bf4993517180045 Mon Sep 17 00:00:00 2001
From: Godson-Gnanaraj <lgodsongnanaraj@gmail.com>
Date: Tue, 25 Oct 2022 08:44:43 +0530
Subject: [PATCH] Replace indentation of source before parsing with ast.

closes #5323
---
 scrapy/utils/misc.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 1221b39b2..c0258c8d9 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -226,7 +226,14 @@ def is_generator_with_return_value(callable):
         return value is None or isinstance(value, ast.NameConstant) and value.value is None
 
     if inspect.isgeneratorfunction(callable):
-        code = re.sub(r"^[\t ]+", "", inspect.getsource(callable))
+        pattern = r"(^[\t ]+)"
+        src = inspect.getsource(callable)
+        match = re.match(pattern, src)  # Find indentation
+        code = re.sub(pattern, "", src)
+        if match:
+            # Remove indentation
+            code = re.sub(f"\n{match.group(0)}", "\n", code)
+
         tree = ast.parse(code)
         for node in walk_callable(tree):
             if isinstance(node, ast.Return) and not returns_none(node):