Add typing to scrapy/utils/misc.py.

scrapy/utils/misc.py

@@ -10,7 +10,21 @@ from contextlib import contextmanager
 from functools import partial
 from importlib import import_module
 from pkgutil import iter_modules
-from typing import TYPE_CHECKING, Any, Callable, Union
+from types import ModuleType
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Deque,
+    Generator,
+    Iterable,
+    List,
+    Optional,
+    Pattern,
+    Union,
+    cast,
+)
 
 from w3lib.html import replace_entities
 
@@ -26,7 +40,7 @@ if TYPE_CHECKING:
 _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
 
 
-def arg_to_iter(arg):
+def arg_to_iter(arg: Any) -> Iterable[Any]:
     """Convert an argument to an iterable. The argument can be a None, single
     value, or an iterable.
 
@@ -35,7 +49,7 @@ def arg_to_iter(arg):
     if arg is None:
         return []
     if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):
-        return arg
+        return cast(Iterable[Any], arg)
     return [arg]
 
 
@@ -72,7 +86,7 @@ def load_object(path: Union[str, Callable]) -> Any:
     return obj
 
 
-def walk_modules(path):
+def walk_modules(path: str) -> List[ModuleType]:
     """Loads a module and all its submodules from the given module path and
     returns them. If *any* module throws an exception while importing, that
     exception is thrown back.
@@ -80,7 +94,7 @@ def walk_modules(path):
     For example: walk_modules('scrapy.utils')
     """
 
-    mods = []
+    mods: List[ModuleType] = []
     mod = import_module(path)
     mods.append(mod)
     if hasattr(mod, "__path__"):
@@ -94,7 +108,9 @@ def walk_modules(path):
     return mods
 
 
-def extract_regex(regex, text, encoding="utf-8"):
+def extract_regex(
+    regex: Union[str, Pattern], text: str, encoding: str = "utf-8"
+) -> List[str]:
     """Extract a list of unicode strings from the given text/encoding using the following policies:
 
     * if the regex contains a named group called "extract" that will be returned
@@ -111,9 +127,11 @@ def extract_regex(regex, text, encoding="utf-8"):
         regex = re.compile(regex, re.UNICODE)
 
     try:
-        strings = [regex.search(text).group("extract")]  # named group
+        # named group
+        strings = [regex.search(text).group("extract")]  # type: ignore[union-attr]
     except Exception:
-        strings = regex.findall(text)  # full regex or numbered groups
+        # full regex or numbered groups
+        strings = regex.findall(text)
     strings = flatten(strings)
 
     if isinstance(text, str):
@@ -123,7 +141,7 @@ def extract_regex(regex, text, encoding="utf-8"):
     ]
 
 
-def md5sum(file):
+def md5sum(file: IO) -> str:
     """Calculate the md5 checksum of a file-like object without reading its
     whole content in memory.
 
@@ -140,7 +158,7 @@ def md5sum(file):
     return m.hexdigest()
 
 
-def rel_has_nofollow(rel):
+def rel_has_nofollow(rel: Optional[str]) -> bool:
     """Return True if link rel attribute has nofollow type"""
     return rel is not None and "nofollow" in rel.replace(",", " ").split()
 
@@ -181,7 +199,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
 
 
 @contextmanager
-def set_environ(**kwargs):
+def set_environ(**kwargs: str) -> Generator[None, Any, None]:
     """Temporarily set environment variables inside the context manager and
     fully restore previous environment afterwards
     """
@@ -198,11 +216,11 @@ def set_environ(**kwargs):
                 os.environ[k] = v
 
 
-def walk_callable(node):
+def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
     """Similar to ``ast.walk``, but walks only function body and skips nested
     functions defined within the node.
     """
-    todo = deque([node])
+    todo: Deque[ast.AST] = deque([node])
     walked_func_def = False
     while todo:
         node = todo.popleft()
@@ -227,7 +245,7 @@ def is_generator_with_return_value(callable: Callable) -> bool:
     if callable in _generator_callbacks_cache:
         return bool(_generator_callbacks_cache[callable])
 
-    def returns_none(return_node):
+    def returns_none(return_node: ast.Return) -> bool:
         value = return_node.value
         return (
             value is None or isinstance(value, ast.NameConstant) and value.value is None

scrapy/utils/python.py

@@ -22,7 +22,7 @@ from typing import (
 from scrapy.utils.asyncgen import as_async_generator
 
 
-def flatten(x):
+def flatten(x: Iterable) -> list:
     """flatten(sequence) -> list
 
     Returns a single, flat list which contains all elements retrieved
@@ -42,7 +42,7 @@ def flatten(x):
     return list(iflatten(x))
 
 
-def iflatten(x):
+def iflatten(x: Iterable) -> Iterable:
     """iflatten(sequence) -> iterator
 
     Similar to ``.flatten()``, but returns iterator instead"""