diff --git a/conftest.py b/conftest.py index 77b0e033b..3af072318 100644 --- a/conftest.py +++ b/conftest.py @@ -57,7 +57,7 @@ def pytest_addoption(parser): def reactor_pytest(request): if not request.cls: # doctests - return + return None request.cls.reactor_pytest = request.config.getoption("--reactor") return request.cls.reactor_pytest diff --git a/pylintrc b/pylintrc index c60e4e16a..e927b903c 100644 --- a/pylintrc +++ b/pylintrc @@ -18,14 +18,12 @@ disable=abstract-method, disallowed-name, duplicate-code, # https://github.com/PyCQA/pylint/issues/214 eval-used, - expression-not-assigned, fixme, function-redefined, global-statement, implicit-str-concat, import-error, import-outside-toplevel, - inconsistent-return-statements, inherit-non-class, invalid-name, invalid-overridden-method, @@ -37,25 +35,20 @@ disable=abstract-method, logging-not-lazy, lost-exception, missing-docstring, - no-else-raise, - no-else-return, no-member, no-method-argument, no-name-in-module, no-self-argument, no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268 not-callable, - pointless-exception-statement, pointless-statement, pointless-string-statement, protected-access, raise-missing-from, - redefined-argument-from-local, redefined-builtin, redefined-outer-name, reimported, signature-differs, - super-init-not-called, too-few-public-methods, too-many-ancestors, too-many-arguments, @@ -73,9 +66,7 @@ disable=abstract-method, unreachable, unused-argument, unused-import, - unused-private-member, unused-variable, - unused-wildcard-import, used-before-assignment, useless-return, wildcard-import, diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 6b6a80bb5..0d71ab6c6 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -23,7 +23,7 @@ class Command(BaseRunSpiderCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) < 1: raise UsageError() - elif len(args) > 1: + if len(args) > 1: raise UsageError( "running 'scrapy crawl' with more than one spider is not supported" ) diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 34313d731..438375e02 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -35,7 +35,8 @@ class Command(ScrapyCommand): try: spidercls = self.crawler_process.spider_loader.load(args[0]) except KeyError: - return self._err(f"Spider not found: {args[0]}") + self._err(f"Spider not found: {args[0]}") + return sfile = sys.modules[spidercls.__module__].__file__ assert sfile diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 2059dcf75..fba294851 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -399,8 +399,7 @@ class Command(BaseRunSpiderCommand): # parse arguments if not len(args) == 1 or not is_url(args[0]): raise UsageError() - else: - url = args[0] + url = args[0] # prepare spidercls self.set_spidercls(url, opts) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index c20c02ca6..9071395e3 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -38,7 +38,9 @@ class Contract: assert cb is not None @wraps(cb) - def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: + def wrapper( # pylint: disable=inconsistent-return-statements + response: Response, **cb_kwargs: Any + ) -> list[Any]: try: results.startTest(self.testcase_pre) self.pre_process(response) @@ -67,7 +69,9 @@ class Contract: assert cb is not None @wraps(cb) - def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: + def wrapper( # pylint: disable=inconsistent-return-statements + response: Response, **cb_kwargs: Any + ) -> list[Any]: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 218f44bbb..20377ac06 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -92,9 +92,8 @@ class DownloadHandlers: ) self._notconfigured[scheme] = str(ex) return None - else: - self._handlers[scheme] = dh - return dh + self._handlers[scheme] = dh + return dh def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: scheme = urlparse_cached(request).scheme diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 70a769771..598659b4d 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -70,7 +70,10 @@ class ReceivedDataProtocol(Protocol): return self.__filename def close(self) -> None: - self.body.close() if self.filename else self.body.seek(0) + if self.filename: + self.body.close() + else: + self.body.seek(0) _CODE_RE = re.compile(r"\d+") diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 60e7adb2f..db4191385 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -73,7 +73,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): ) -> Generator[Deferred[Any], Any, Response | Request]: if response is None: raise TypeError("Received None in process_response") - elif isinstance(response, Request): + if isinstance(response, Request): return response for method in self.methods["process_response"]: diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index d056a00ba..60cffae35 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -172,7 +172,7 @@ class ExecutionEngine: assert self.spider is not None # typing if self.paused: - return None + return while ( not self._needs_backout() @@ -418,7 +418,7 @@ class ExecutionEngine: if isinstance(x, Failure) and isinstance(x.value, ex) } if DontCloseSpider in detected_ex: - return None + return if self.spider_is_idle(): ex = detected_ex.get(CloseSpider, CloseSpider(reason="finished")) assert isinstance(ex, CloseSpider) # typing diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index bebee1236..f09d1903c 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -312,8 +312,7 @@ class Scheduler(BaseScheduler): assert self.stats is not None self.stats.inc_value("scheduler/unserializable", spider=self.spider) return False - else: - return True + return True def _mqpush(self, request: Request) -> None: self.mqs.push(request) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index f7947d35d..a63ee40bf 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -174,14 +174,13 @@ class SpiderMiddlewareManager(MiddlewareManager): # _process_spider_exception too, which complicates the architecture msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" raise _InvalidOutput(msg) - elif result is None: + if result is None: continue - else: - msg = ( - f"{method.__qualname__} must return None " - f"or an iterable, got {type(result)}" - ) - raise _InvalidOutput(msg) + msg = ( + f"{method.__qualname__} must return None " + f"or an iterable, got {type(result)}" + ) + raise _InvalidOutput(msg) return _failure # This method cannot be made async def, as _process_spider_exception relies on the Deferred result diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 6ab88dbb4..af06b7790 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -105,7 +105,7 @@ class ItemFilter: class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" - def __init__(uri, *, feed_options=None): + def __init__(uri, *, feed_options=None): # pylint: disable=super-init-not-called """Initialize the storage with the parameters given in the URI and the feed-specific options (see :setting:`FEEDS`)""" diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 29743565d..10ad1305e 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -152,8 +152,7 @@ def _get_form( form = forms[formnumber] except IndexError: raise IndexError(f"Form number {formnumber} not found in {response}") - else: - return cast(FormElement, form) + return cast(FormElement, form) def _get_inputs( @@ -264,5 +263,4 @@ def _get_clickable( f"Multiple elements found ({el!r}) matching the " f"criteria in clickdata: {clickdata!r}" ) - else: - raise ValueError(f"No clickable element matching clickdata: {clickdata!r}") + raise ValueError(f"No clickable element matching clickdata: {clickdata!r}") diff --git a/scrapy/mail.py b/scrapy/mail.py index ce7beb773..10dc7fed2 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -117,8 +117,8 @@ class MailSender: if charset: msg.set_charset(charset) msg.attach(MIMEText(body, "plain", charset or "us-ascii")) - for attach_name, mimetype, f in attachs: - part = MIMEBase(*mimetype.split("/")) + for attach_name, attach_mimetype, f in attachs: + part = MIMEBase(*attach_mimetype.split("/")) part.set_payload(f.read()) Encoders.encode_base64(part) part.add_header( diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 4a8639c22..73cf37d28 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -265,8 +265,7 @@ class S3FilesStore: kwarg = mapping[key] except KeyError: raise TypeError(f'Header "{key}" is not supported by botocore') - else: - extra[kwarg] = value + extra[kwarg] = value return extra diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 99a6cc5f6..0e8260736 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -141,9 +141,8 @@ class CachingHostnameResolver: addressTypes, transportSemantics, ) - else: - resolutionReceiver.resolutionBegan(HostResolution(hostName)) - for addr in addresses: - resolutionReceiver.addressResolved(addr) - resolutionReceiver.resolutionComplete() - return resolutionReceiver + resolutionReceiver.resolutionBegan(HostResolution(hostName)) + for addr in addresses: + resolutionReceiver.addressResolved(addr) + resolutionReceiver.resolutionComplete() + return resolutionReceiver diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 274ced3e3..3ebdb351a 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -501,11 +501,9 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): except KeyError: if default is self.__default: raise - return default - else: - self.__delitem__(name) - return value + self.__delitem__(name) + return value class Settings(BaseSettings): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index bdf1f168a..720217c97 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -325,9 +325,8 @@ def _load_policy_class( msg = f"Could not load referrer policy {policy!r}" if not warning_only: raise RuntimeError(msg) - else: - warnings.warn(msg, RuntimeWarning) - return None + warnings.warn(msg, RuntimeWarning) + return None class RefererMiddleware: diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 91a49c652..e621525f2 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -44,8 +44,7 @@ def build_component_list( "convert to the same " "object, please update your settings" ) - else: - compbs.set(convert(k), v, priority=prio) + compbs.set(convert(k), v, priority=prio) return compbs _check_components(compdict) return {convert(k): v for k, v in compdict.items()} diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 6b9b4114f..95844a48c 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -101,7 +101,7 @@ DEFAULT_PYTHON_SHELLS: KnownShellsT = { def get_shell_embed_func( shells: Iterable[str] | None = None, known_shells: KnownShellsT | None = None -) -> Any: +) -> EmbedFuncT | None: """Return the first acceptable shell-embed function from a given list of shell names. """ @@ -117,6 +117,7 @@ def get_shell_embed_func( return known_shells[shell]() except ImportError: continue + return None def start_python_console( diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 9ca6c6a24..9f1b816c8 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -109,8 +109,7 @@ def mustbe_deferred( return defer_fail(failure.Failure(e)) except Exception: return defer_fail(failure.Failure()) - else: - return defer_result(result) + return defer_result(result) def parallel( diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 596cf89e4..39f46270b 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -36,11 +36,10 @@ def _colorize(text: str, colorize: bool = True) -> str: from pygments import highlight except ImportError: return text - else: - from pygments.formatters import TerminalFormatter - from pygments.lexers import PythonLexer + from pygments.formatters import TerminalFormatter + from pygments.lexers import PythonLexer - return highlight(text, PythonLexer(), TerminalFormatter()) + return highlight(text, PythonLexer(), TerminalFormatter()) def pformat(obj: Any, *args: Any, **kwargs: Any) -> str: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index c3808426a..a40b835cd 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -51,6 +51,7 @@ class TopLevelFormatter(logging.Filter): """ def __init__(self, loggers: list[str] | None = None): + super().__init__() self.loggers: list[str] = loggers or [] def filter(self, record: logging.LogRecord) -> bool: diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index d970f5da5..3864d054f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -323,9 +323,8 @@ def without_none_values( """ if isinstance(iterable, Mapping): return {k: v for k, v in iterable.items() if v is not None} - else: - # the iterable __init__ must take another iterable - return type(iterable)(v for v in iterable if v is not None) # type: ignore[call-arg] + # the iterable __init__ must take another iterable + return type(iterable)(v for v in iterable if v is not None) # type: ignore[call-arg] def global_object_name(obj: Any) -> str: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index e7bd0b232..ac4358410 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -26,7 +26,7 @@ if TYPE_CHECKING: _T = TypeVar("_T") -def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] +def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # pylint: disable=inconsistent-return-statements """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index ecc83d1c8..7c8ca51f2 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -66,9 +66,8 @@ def _remove_html_comments(body: bytes) -> bytes: end = body.find(b"-->", start + 1) if end == -1: return body[:start] - else: - body = body[:start] + body[end + 3 :] - start = body.find(b"