Typing for build_from_*. (#6326)

scrapy/core/engine.py

@@ -366,7 +366,8 @@ class ExecutionEngine:
         self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
         self.spider = spider
         if hasattr(scheduler, "open"):
-            yield scheduler.open(spider)
+            if d := scheduler.open(spider):
+                yield d
         yield self.scraper.open_spider(spider)
         assert self.crawler.stats
         self.crawler.stats.open_spider(spider)

scrapy/core/scheduler.py

@@ -322,6 +322,7 @@ class Scheduler(BaseScheduler):
 
     def _mq(self):
         """Create a new priority queue instance, with in-memory storage"""
+        assert self.crawler
         return build_from_crawler(
             self.pqclass,
             self.crawler,
@@ -331,6 +332,7 @@ class Scheduler(BaseScheduler):
 
     def _dq(self):
         """Create a new priority queue instance, with disk storage"""
+        assert self.crawler
         assert self.dqdir
         state = self._read_dqs_state(self.dqdir)
         q = build_from_crawler(

scrapy/crawler.py

@@ -445,7 +445,9 @@ class CrawlerProcess(CrawlerRunner):
             d.addBoth(self._stop_reactor)
 
         resolver_class = load_object(self.settings["DNS_RESOLVER"])
-        resolver = build_from_crawler(resolver_class, self, reactor=reactor)
+        # We pass self, which is CrawlerProcess, instead of Crawler here,
+        # which works because the default resolvers only use crawler.settings.
+        resolver = build_from_crawler(resolver_class, self, reactor=reactor)  # type: ignore[arg-type]
         resolver.install_on_reactor()
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))

scrapy/utils/misc.py

@@ -1,5 +1,7 @@
 """Helper functions which don't fit anywhere else"""
 
+from __future__ import annotations
+
 import ast
 import hashlib
 import inspect
@@ -22,6 +24,8 @@ from typing import (
     Iterable,
     List,
     Optional,
+    Type,
+    TypeVar,
     Union,
     cast,
 )
@@ -32,9 +36,11 @@ from scrapy.utils.datatypes import LocalWeakReferencedCache
 
 if TYPE_CHECKING:
     from scrapy import Spider
-
+    from scrapy.crawler import Crawler
+    from scrapy.settings import BaseSettings
 
 _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
+T = TypeVar("T")
 
 
 def arg_to_iter(arg: Any) -> Iterable[Any]:
@@ -177,7 +183,9 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
     return instance
 
 
-def build_from_crawler(objcls, crawler, /, *args, **kwargs):
+def build_from_crawler(
+    objcls: Type[T], crawler: Crawler, /, *args: Any, **kwargs: Any
+) -> T:
     """Construct a class instance using its ``from_crawler`` constructor.
 
     ``*args`` and ``**kwargs`` are forwarded to the constructor.
@@ -185,20 +193,22 @@ def build_from_crawler(objcls, crawler, /, *args, **kwargs):
     Raises ``TypeError`` if the resulting instance is ``None``.
     """
     if hasattr(objcls, "from_crawler"):
-        instance = objcls.from_crawler(crawler, *args, **kwargs)
+        instance = objcls.from_crawler(crawler, *args, **kwargs)  # type: ignore[attr-defined]
         method_name = "from_crawler"
     elif hasattr(objcls, "from_settings"):
-        instance = objcls.from_settings(crawler.settings, *args, **kwargs)
+        instance = objcls.from_settings(crawler.settings, *args, **kwargs)  # type: ignore[attr-defined]
         method_name = "from_settings"
     else:
         instance = objcls(*args, **kwargs)
         method_name = "__new__"
     if instance is None:
         raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
-    return instance
+    return cast(T, instance)
 
 
-def build_from_settings(objcls, settings, /, *args, **kwargs):
+def build_from_settings(
+    objcls: Type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any
+) -> T:
     """Construct a class instance using its ``from_settings`` constructor.
 
     ``*args`` and ``**kwargs`` are forwarded to the constructor.
@@ -206,14 +216,14 @@ def build_from_settings(objcls, settings, /, *args, **kwargs):
     Raises ``TypeError`` if the resulting instance is ``None``.
     """
     if hasattr(objcls, "from_settings"):
-        instance = objcls.from_settings(settings, *args, **kwargs)
+        instance = objcls.from_settings(settings, *args, **kwargs)  # type: ignore[attr-defined]
         method_name = "from_settings"
     else:
         instance = objcls(*args, **kwargs)
         method_name = "__new__"
     if instance is None:
         raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
-    return instance
+    return cast(T, instance)
 
 
 @contextmanager
@@ -290,7 +300,7 @@ def is_generator_with_return_value(callable: Callable) -> bool:
     return bool(_generator_callbacks_cache[callable])
 
 
-def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None:
+def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None:
     """
     Logs a warning if a callable is a generator function and includes
     a 'return' statement with a value different than None