diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index aa79cbc0d..ee0cb4b1e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -11,7 +11,7 @@ jobs: - python-version: "3.11" env: TOXENV: pylint - - python-version: 3.7 + - python-version: 3.8 env: TOXENV: typing - python-version: "3.11" # Keep in sync with .readthedocs.yml diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7d27a5193..0b179da32 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -94,7 +94,7 @@ class Downloader: dfd = self.middleware.download(self._enqueue_request, request, spider) return dfd.addBoth(_deactivate) - def needs_backout(self): + def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency def _get_slot(self, request, spider): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 1efbdb271..c6738b531 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -7,13 +7,24 @@ For more information see docs/topics/architecture.rst import logging import warnings from time import time -from typing import Callable, Iterable, Iterator, Optional, Set, Union +from typing import ( + Any, + Callable, + Generator, + Iterable, + Iterator, + Optional, + Set, + Union, + cast, +) from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals +from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning from scrapy.http import Request, Response @@ -75,7 +86,7 @@ class ExecutionEngine: self.paused = False self.scheduler_cls = self._get_scheduler_class(crawler.settings) downloader_cls = load_object(self.settings["DOWNLOADER"]) - self.downloader = downloader_cls(crawler) + self.downloader: Downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback = spider_closed_callback @@ -91,20 +102,20 @@ class ExecutionEngine: return scheduler_cls @inlineCallbacks - def start(self) -> Deferred: + def start(self) -> Generator[Deferred, Any, None]: if self.running: raise RuntimeError("Engine already running") self.start_time = time() yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True - self._closewait = Deferred() + self._closewait: Deferred = Deferred() yield self._closewait def stop(self) -> Deferred: """Gracefully stop the execution engine""" @inlineCallbacks - def _finish_stopping_engine(_) -> Deferred: + def _finish_stopping_engine(_) -> Generator[Deferred, Any, None]: yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) self._closewait.callback(None) @@ -172,11 +183,13 @@ class ExecutionEngine: self._spider_idle() def _needs_backout(self) -> bool: + assert self.slot is not None # typing + assert self.scraper.slot is not None # typing return ( not self.running - or self.slot.closing # type: ignore[union-attr] + or bool(self.slot.closing) or self.downloader.needs_backout() - or self.scraper.slot.needs_backout() # type: ignore[union-attr] + or self.scraper.slot.needs_backout() ) def _next_request_from_scheduler(self) -> Optional[Deferred]: @@ -196,7 +209,7 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - d.addBoth(lambda _: self.slot.remove_request(request)) + d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request)) d.addErrback( lambda f: logger.info( "Error while removing request from slot", @@ -424,7 +437,7 @@ class ExecutionEngine: dfd.addErrback(log_failure("Scraper close failure")) if hasattr(self.slot.scheduler, "close"): - dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) + dfd.addBoth(lambda _: cast(Slot, self.slot).scheduler.close(reason)) dfd.addErrback(log_failure("Scheduler close failure")) dfd.addBoth( diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 1c43d241c..215ea9716 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -6,7 +6,12 @@ from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.endpoints import HostnameEndpoint from twisted.python.failure import Failure -from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory +from twisted.web.client import ( + URI, + BrowserLikePolicyForHTTPS, + ResponseFailed, + _StandardEndpointFactory, +) from twisted.web.error import SchemeNotSupported from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory @@ -35,7 +40,7 @@ class H2ConnectionPool: # Received a request while connecting to remote # Create a deferred which will fire with the H2ClientProtocol # instance - d = Deferred() + d: Deferred = Deferred() self._pending_requests[key].append(d) return d @@ -53,14 +58,14 @@ class H2ConnectionPool: ) -> Deferred: self._pending_requests[key] = deque() - conn_lost_deferred = Deferred() + conn_lost_deferred: Deferred = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) conn_d = endpoint.connect(factory) conn_d.addCallback(self.put_connection, key) - d = Deferred() + d: Deferred = Deferred() self._pending_requests[key].append(d) return d @@ -83,7 +88,7 @@ class H2ConnectionPool: pending_requests = self._pending_requests.pop(key, None) while pending_requests: d = pending_requests.popleft() - d.errback(errors) + d.errback(ResponseFailed(errors)) def close_connections(self) -> None: """Close all the HTTP/2 connections and remove them from pool @@ -92,6 +97,7 @@ class H2ConnectionPool: Deferred that fires when all connections have been closed """ for conn in self._connections.values(): + assert conn.transport is not None # typing conn.transport.abortConnection() diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 0bf69e513..bc8da50d7 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -140,6 +140,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection """ + assert self.transport is not None # typing return bool(self.transport.connected) and self.metadata["settings_acknowledged"] @property @@ -196,6 +197,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """Write data to the underlying transport connection from the HTTP2 connection instance if any """ + assert self.transport is not None # typing # Reset the idle timeout as connection is still actively sending data self.resetTimeout() @@ -226,6 +228,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # Initialize the timeout self.setTimeout(self.IDLE_TIMEOUT) + assert self.transport is not None # typing destination = self.transport.getPeer() self.metadata["ip_address"] = ipaddress.ip_address(destination.host) @@ -237,12 +240,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """Helper function to lose the connection with the error sent as a reason""" self._conn_lost_errors += errors + assert self.transport is not None # typing self.transport.loseConnection() def handshakeCompleted(self) -> None: """ Close the connection if it's not made via the expected protocol """ + assert self.transport is not None # typing if ( self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME @@ -275,6 +280,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # hyper-h2 does not drop the connection in this scenario, we # need to abort the connection manually. self._conn_lost_errors += [e] + assert self.transport is not None # typing self.transport.abortConnection() return @@ -388,6 +394,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._send_pending_requests() # Update certificate when our HTTP/2 connection is established + assert self.transport is not None # typing self.metadata["certificate"] = Certificate(self.transport.getPeerCertificate()) def stream_ended(self, event: StreamEnded) -> None: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 87beb41e5..6c6ed6f9b 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -151,7 +151,7 @@ class Stream: else: self.close(StreamCloseReason.CANCELLED) - self._deferred_response = Deferred(_cancel) + self._deferred_response: Deferred = Deferred(_cancel) def __repr__(self) -> str: return f"Stream(id={self.stream_id!r})" diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 1a09f22f7..8468b8419 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -64,7 +64,7 @@ class Slot: def add_response_request( self, result: Union[Response, Failure], request: Request ) -> Deferred: - deferred = Deferred() + deferred: Deferred = Deferred() self.queue.append((result, request, deferred)) if isinstance(result, Response): self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -203,10 +203,12 @@ class Scraper: callback=callback, callbackKeywords=result.request.cb_kwargs ) else: # result is a Failure - result.request = request + # TODO: properly type adding this attribute to a Failure + result.request = request # type: ignore[attr-defined] warn_on_generator_with_return_value(spider, request.errback) dfd = defer_fail(result) - dfd.addErrback(request.errback) + if request.errback: + dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) def handle_spider_error( diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 971cf52e4..c21985b18 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -155,7 +155,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # simplified when downgrading is removed. if dfd.called: # the result is available immediately if _process_spider_output didn't do downgrading - return dfd.result + return cast(MutableChain, dfd.result) # we forbid waiting here because otherwise we would need to return a deferred from # _process_spider_exception too, which complicates the architecture msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" @@ -180,7 +180,7 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, result: Union[Iterable, AsyncIterable], start_index: int = 0, - ) -> Deferred: + ) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered: Union[MutableChain, MutableAsyncChain] diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index bdc6a3e39..8da779b7c 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,10 +5,17 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union +from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit -from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement +from lxml.html import ( + FormElement, + HTMLParser, + InputElement, + MultipleSelectOptions, + SelectElement, + TextareaElement, +) from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace @@ -19,7 +26,8 @@ from scrapy.utils.response import get_base_url FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") -FormdataType = Optional[Union[dict, List[Tuple[str, str]]]] +FormdataKVType = Tuple[str, Union[str, Iterable[str]]] +FormdataType = Optional[Union[dict, List[FormdataKVType]]] class FormRequest(Request): @@ -50,7 +58,7 @@ class FormRequest(Request): response: TextResponse, formname: Optional[str] = None, formid: Optional[str] = None, - formnumber: Optional[int] = 0, + formnumber: int = 0, formdata: FormdataType = None, clickdata: Optional[dict] = None, dont_click: bool = False, @@ -79,6 +87,7 @@ class FormRequest(Request): def _get_form_url(form: FormElement, url: Optional[str]) -> str: + assert form.base_url is not None # typing if url is None: action = form.get("action") if action is None: @@ -87,11 +96,11 @@ def _get_form_url(form: FormElement, url: Optional[str]) -> str: return urljoin(form.base_url, url) -def _urlencode(seq: Iterable, enc: str) -> str: +def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str: values = [ (to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq - for v in (vs if is_listlike(vs) else [vs]) + for v in (cast(Iterable[str], vs) if is_listlike(vs) else [cast(str, vs)]) ] return urlencode(values, doseq=True) @@ -100,7 +109,7 @@ def _get_form( response: TextResponse, formname: Optional[str], formid: Optional[str], - formnumber: Optional[int], + formnumber: int, formxpath: Optional[str], ) -> FormElement: """Find the wanted form element within the given response.""" @@ -133,13 +142,12 @@ def _get_form( raise ValueError(f"No
element found with {formxpath}") # If we get here, it means that either formname was None or invalid - if formnumber is not None: - try: - form = forms[formnumber] - except IndexError: - raise IndexError(f"Form number {formnumber} not found in {response}") - else: - return form + try: + form = forms[formnumber] + except IndexError: + raise IndexError(f"Form number {formnumber} not found in {response}") + else: + return form def _get_inputs( @@ -147,7 +155,7 @@ def _get_inputs( formdata: FormdataType, dont_click: bool, clickdata: Optional[dict], -) -> List[Tuple[str, str]]: +) -> List[FormdataKVType]: """Return a list of key-value pairs for the inputs found in the given form.""" try: formdata_keys = dict(formdata or ()).keys() @@ -165,7 +173,7 @@ def _get_inputs( ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={"re": "http://exslt.org/regular-expressions"}, ) - values = [ + values: List[FormdataKVType] = [ (k, "" if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys @@ -183,27 +191,25 @@ def _get_inputs( return values -def _value(ele: HtmlElement): +def _value( + ele: Union[InputElement, SelectElement, TextareaElement] +) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: n = ele.name v = ele.value if ele.tag == "select": - return _select_value(ele, n, v) + return _select_value(cast(SelectElement, ele), n, v) return n, v -def _select_value(ele: SelectElement, n: str, v: str): +def _select_value( + ele: SelectElement, n: Optional[str], v: Union[None, str, MultipleSelectOptions] +) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected # And for select tags without options o = ele.value_options return (n, o[0]) if o else (None, None) - if v is not None and multiple: - # This is a workaround to bug in lxml fixed 2.3.1 - # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 - selected_options = ele.xpath(".//option[@selected]") - values = [(o.get("value") or o.text or "").strip() for o in selected_options] - return n, values return n, v diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index ec130d685..d25ebbdf4 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -17,6 +17,7 @@ from typing import ( List, Optional, Union, + cast, ) from twisted.internet import defer @@ -38,7 +39,7 @@ def defer_fail(_failure: Failure) -> Deferred: """ from twisted.internet import reactor - d = Deferred() + d: Deferred = Deferred() reactor.callLater(0.1, d.errback, _failure) return d @@ -52,7 +53,7 @@ def defer_succeed(result) -> Deferred: """ from twisted.internet import reactor - d = Deferred() + d: Deferred = Deferred() reactor.callLater(0.1, d.callback, result) return d @@ -84,7 +85,7 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: def parallel( iterable: Iterable, count: int, callable: Callable, *args, **named -) -> DeferredList: +) -> Deferred: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -182,7 +183,9 @@ class _AsyncCooperatorAdapter(Iterator): def _call_anext(self) -> None: # This starts waiting for the next result from aiterator. # If aiterator is exhausted, _errback will be called. - self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) + self.anext_deferred = cast( + Deferred, deferred_from_coro(self.aiterator.__anext__()) + ) self.anext_deferred.addCallbacks(self._callback, self._errback) def __next__(self) -> Deferred: @@ -190,7 +193,7 @@ class _AsyncCooperatorAdapter(Iterator): # It also calls __anext__() if needed. if self.finished: raise StopIteration - d = Deferred() + d: Deferred = Deferred() self.waiting_deferreds.append(d) if not self.anext_deferred: self._call_anext() @@ -199,17 +202,17 @@ class _AsyncCooperatorAdapter(Iterator): def parallel_async( async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named -) -> DeferredList: +) -> Deferred: """Like parallel but for async iterators""" coop = Cooperator() work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) - dl = DeferredList([coop.coiterate(work) for _ in range(count)]) + dl: Deferred = DeferredList([coop.coiterate(work) for _ in range(count)]) return dl def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred built by chaining the given callbacks""" - d = Deferred() + d: Deferred = Deferred() for x in callbacks: d.addCallback(x, *a, **kw) d.callback(input) @@ -220,7 +223,7 @@ def process_chain_both( callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw ) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" - d = Deferred() + d: Deferred = Deferred() for cb, eb in zip(callbacks, errbacks): d.addCallbacks( callback=cb, @@ -242,7 +245,7 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) + d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d @@ -285,7 +288,7 @@ def deferred_from_coro(o) -> Any: if not is_asyncio_reactor_installed(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" - return ensureDeferred(o) + return ensureDeferred(cast(Coroutine[Deferred, Any, Any], o)) # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor event_loop = _get_asyncio_event_loop() return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 1771e5550..fc50e0f12 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,7 +8,7 @@ import sys import weakref from functools import partial, wraps from itertools import chain -from typing import AsyncGenerator, AsyncIterable, Iterable, Union +from typing import Any, AsyncGenerator, AsyncIterable, Iterable, Union from scrapy.utils.asyncgen import as_async_generator @@ -45,7 +45,7 @@ def iflatten(x): yield el -def is_listlike(x): +def is_listlike(x: Any) -> bool: """ >>> is_listlike("foo") False diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 730d005e8..794678c48 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -6,7 +6,7 @@ import os import re import tempfile import webbrowser -from typing import Any, Callable, Iterable, Optional, Tuple, Union +from typing import Any, Callable, Iterable, Tuple, Union from weakref import WeakKeyDictionary from twisted.web import http @@ -37,12 +37,13 @@ _metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[floa def get_meta_refresh( response: "scrapy.http.response.text.TextResponse", - ignore_tags: Optional[Iterable[str]] = ("script", "noscript"), + ignore_tags: Iterable[str] = ("script", "noscript"), ) -> Union[Tuple[None, None], Tuple[float, str]]: """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] - _metaref_cache[response] = html.get_meta_refresh( + # a w3lib typing bug here, fixed in https://github.com/scrapy/w3lib/pull/211 + _metaref_cache[response] = html.get_meta_refresh( # type: ignore[assignment] text, response.url, response.encoding, ignore_tags=ignore_tags ) return _metaref_cache[response] diff --git a/setup.cfg b/setup.cfg index 1fab6fe22..db79c5821 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,6 +6,12 @@ universal=1 [mypy] ignore_missing_imports = true + +# Interface classes are hard to support +[mypy-twisted.internet.interfaces] +follow_imports = skip + +[mypy-twisted.internet.reactor] follow_imports = skip # FIXME: remove the following sections once the issues are solved diff --git a/tests/mockserver.py b/tests/mockserver.py index d541f3f7c..eb4f8334e 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -10,6 +10,7 @@ from urllib.parse import urlencode from OpenSSL import SSL from twisted.internet import defer, reactor, ssl +from twisted.internet.protocol import ServerFactory from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory @@ -368,6 +369,8 @@ if __name__ == "__main__": ) args = parser.parse_args() + factory: ServerFactory + if args.type == "http": root = Root() factory = Site(root) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index f9b2d853c..a39ed0694 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -37,7 +37,3 @@ class ScrapyUtilsTest(unittest.TestCase): pinned_twisted_version_string = match[1] self.assertEqual(twisted_version.short(), pinned_twisted_version_string) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 9ec51f75b..0ea1ef5eb 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -13,7 +13,7 @@ spider = Spider("foo") class TestHttpProxyMiddleware(TestCase): - failureException = AssertionError + failureException = AssertionError # type: ignore[assignment] def setUp(self): self._oldenv = os.environ.copy() diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 17a94f036..4709007a8 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -5,6 +5,7 @@ import shutil import string from ipaddress import IPv4Address from pathlib import Path +from typing import Dict from unittest import mock, skipIf from urllib.parse import urlencode @@ -87,6 +88,7 @@ class GetDataHtmlLarge(LeafResource): class PostDataJsonMixin: @staticmethod def make_response(request: TxRequest, extra_data: str): + assert request.content is not None response = { "request-headers": {}, "request-body": json.loads(request.content.read()), @@ -145,7 +147,8 @@ class QueryParams(LeafResource): request.setHeader("Content-Type", "application/json; charset=UTF-8") request.setHeader("Content-Encoding", "UTF-8") - query_params = {} + query_params: Dict[str, str] = {} + assert request.args is not None for k, v in request.args.items(): query_params[str(k, "utf-8")] = str(v[0], "utf-8") diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 974a0023d..d167adbb7 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -112,6 +112,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): Should work for process_spider_output and, when it's supported, process_start_requests. """ + ITEM_TYPE: type RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects @staticmethod diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index bab6e9b68..faf7d2709 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -527,7 +527,3 @@ class TestHelper(unittest.TestCase): type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}" ) self.assertEqual(a, b) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index fbf60ca71..57f40c2e5 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -257,7 +257,3 @@ class UtilsPythonTestCase(unittest.TestCase): without_none_values({"one": 1, "none": None, "three": 3, "four": 4}), {"one": 1, "three": 3, "four": 4}, ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tox.ini b/tox.ini index 77385968e..5a9d9cf29 100644 --- a/tox.ini +++ b/tox.ini @@ -37,11 +37,13 @@ install_command = [testenv:typing] basepython = python3 deps = - lxml-stubs==0.4.0 - mypy==0.991 + mypy==1.0.1 types-attrs==19.1.0 - types-pyOpenSSL==23.0.0.2 - types-setuptools==65.7.0.3 + types-lxml==2023.2.11 + types-Pillow==9.4.0.16 + types-Pygments==2.14.0.5 + types-pyOpenSSL==23.0.0.4 + types-setuptools==67.4.0.1 commands = mypy --show-error-codes {posargs: scrapy tests}