diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index c23a89089..9b63f39f6 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,3 +1,4 @@ +# pylint: disable=import-error from operator import itemgetter from docutils import nodes diff --git a/docs/conf.py b/docs/conf.py index 7a5166053..d06828bcc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,6 +8,8 @@ # # All configuration values have a default; values that are commented out # serve to show the default. + +# pylint: disable=import-error import os import sys from pathlib import Path diff --git a/pyproject.toml b/pyproject.toml index ad85e5c75..8c985753f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,44 +124,34 @@ extension-pkg-allow-list=[ ] [tool.pylint."MESSAGES CONTROL"] +enable = [ + "useless-suppression", +] disable = [ - "abstract-method", - "arguments-differ", - "arguments-renamed", + # Ones we want to ignore "attribute-defined-outside-init", "broad-exception-caught", "consider-using-with", "cyclic-import", - "dangerous-default-value", "disallowed-name", - "duplicate-code", # https://github.com/PyCQA/pylint/issues/214 - "eval-used", + "duplicate-code", # https://github.com/pylint-dev/pylint/issues/214 "fixme", - "import-error", "import-outside-toplevel", - "inherit-non-class", + "inherit-non-class", # false positives with create_deprecated_class() "invalid-name", "invalid-overridden-method", - "isinstance-second-argument-not-valid-type", - "keyword-arg-before-vararg", + "isinstance-second-argument-not-valid-type", # false positives with create_deprecated_class() "line-too-long", "logging-format-interpolation", "logging-fstring-interpolation", "logging-not-lazy", "missing-docstring", "no-member", - "no-method-argument", - "no-name-in-module", - "no-self-argument", - "no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268 + "no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268 "not-callable", - "pointless-statement", - "pointless-string-statement", "protected-access", - "raise-missing-from", "redefined-builtin", "redefined-outer-name", - "signature-differs", "too-few-public-methods", "too-many-ancestors", "too-many-arguments", @@ -173,14 +163,23 @@ disable = [ "too-many-positional-arguments", "too-many-public-methods", "too-many-return-statements", - "unbalanced-tuple-unpacking", - "unnecessary-dunder-call", "unused-argument", "unused-import", "unused-variable", - "used-before-assignment", - "useless-return", + "useless-return", # https://github.com/pylint-dev/pylint/issues/6530 "wrong-import-position", + + # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") + "abstract-method", + "arguments-differ", + "arguments-renamed", + "dangerous-default-value", + "keyword-arg-before-vararg", + "pointless-statement", + "raise-missing-from", + "unbalanced-tuple-unpacking", + "unnecessary-dunder-call", + "used-before-assignment", ] [tool.pytest.ini_options] @@ -222,6 +221,8 @@ extend-select = [ "D", # flake8-future-annotations "FA", + # flynt + "FLY", # refurb "FURB", # isort @@ -238,6 +239,8 @@ extend-select = [ "PIE", # pylint "PL", + # flake8-use-pathlib + "PTH", # flake8-pyi "PYI", # flake8-quotes @@ -246,8 +249,12 @@ extend-select = [ "RET", # flake8-raise "RSE", + # Ruff-specific rules + "RUF", # flake8-bandit "S", + # flake8-simplify + "SIM", # flake8-slots "SLOT", # flake8-debugger @@ -262,22 +269,8 @@ extend-select = [ "YTT", ] ignore = [ - # Assigning to `os.environ` doesn't clear the environment. - "B003", - # Do not use mutable data structures for argument defaults. - "B006", - # Loop control variable not used within the loop body. - "B007", - # Do not perform function calls in argument defaults. - "B008", - # Star-arg unpacking after a keyword argument is strongly discouraged. - "B026", - # Found useless expression. - "B018", - # No explicit stacklevel argument found. - "B028", - # Within an `except` clause, raise exceptions with `raise ... from` - "B904", + # Ones we want to ignore + # Missing docstring in public module "D100", # Missing docstring in public class @@ -324,12 +317,45 @@ ignore = [ "PLR2004", # `for` loop variable overwritten by assignment target "PLW2901", + # String contains ambiguous {}. + "RUF001", + # Docstring contains ambiguous {}. + "RUF002", + # Comment contains ambiguous {}. + "RUF003", + # Mutable class attributes should be annotated with `typing.ClassVar` + "RUF012", # Use of `assert` detected; needed for mypy "S101", # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180 "S321", # Argument default set to insecure SSL protocol "S503", + # Use a context manager for opening files + "SIM115", + # Yoda condition detected + "SIM300", + + # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") + + # Assigning to `os.environ` doesn't clear the environment. + "B003", + # Do not use mutable data structures for argument defaults. + "B006", + # Loop control variable not used within the loop body. + "B007", + # Do not perform function calls in argument defaults. + "B008", + # Found useless expression. + "B018", + # Star-arg unpacking after a keyword argument is strongly discouraged. + "B026", + # No explicit stacklevel argument found. + "B028", + # Within an `except` clause, raise exceptions with `raise ... from` + "B904", + # Use capitalized environment variable + "SIM112", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/__init__.py b/scrapy/__init__.py index c19710a6a..256504c9c 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -13,14 +13,14 @@ from scrapy.selector import Selector from scrapy.spiders import Spider __all__ = [ + "Field", + "FormRequest", + "Item", + "Request", + "Selector", + "Spider", "__version__", "version_info", - "Spider", - "Request", - "FormRequest", - "Selector", - "Item", - "Field", ] diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 9a24871de..48f462c65 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -90,12 +90,10 @@ def _get_commands_dict( def _pop_command_name(argv: list[str]) -> str | None: - i = 0 - for arg in argv[1:]: + for i, arg in enumerate(argv[1:]): if not arg.startswith("-"): del argv[i] return arg - i += 1 return None diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 86d4cc41c..184bd5ca4 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -39,9 +39,8 @@ class Command(BaseRunSpiderCommand): else: self.crawler_process.start() - if ( - self.crawler_process.bootstrap_failed - or hasattr(self.crawler_process, "has_exception") + if self.crawler_process.bootstrap_failed or ( + hasattr(self.crawler_process, "has_exception") and self.crawler_process.has_exception ): self.exitcode = 1 diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index d7dc104c2..2a1dea997 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -154,7 +154,7 @@ class Command(ScrapyCommand): spiders_dir = Path(spiders_module.__file__).parent.resolve() else: spiders_module = None - spiders_dir = Path(".") + spiders_dir = Path() spider_file = f"{spiders_dir / module}.py" shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index cc5c1350b..61aea3ee4 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -174,13 +174,12 @@ class Command(BaseRunSpiderCommand): display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour) def print_requests(self, lvl: int | None = None, colour: bool = True) -> None: - if lvl is None: - if self.requests: - requests = self.requests[max(self.requests)] - else: - requests = [] - else: + if lvl is not None: requests = self.requests.get(lvl, []) + elif self.requests: + requests = self.requests[max(self.requests)] + else: + requests = [] print("# Requests ", "-" * 65) display.pprint(requests, colorize=colour) @@ -269,7 +268,7 @@ class Command(BaseRunSpiderCommand): assert self.crawler_process assert self.spidercls self.crawler_process.crawl(self.spidercls, **opts.spargs) - self.pcrawler = list(self.crawler_process.crawlers)[0] + self.pcrawler = next(iter(self.crawler_process.crawlers)) self.crawler_process.start() if not self.first_response: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index bf8e41020..357ca8b37 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -20,7 +20,7 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType: if abspath.suffix not in (".py", ".pyw"): raise ValueError(f"Not a Python source file: {abspath}") dirname = str(abspath.parent) - sys.path = [dirname] + sys.path + sys.path = [dirname, *sys.path] try: module = import_module(abspath.stem) finally: diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 5cb73f0d2..1adc1530f 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import re import string from importlib.util import find_spec @@ -28,9 +27,9 @@ TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") -def _make_writable(path: str | os.PathLike) -> None: - current_permissions = os.stat(path).st_mode - os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION) +def _make_writable(path: Path) -> None: + current_permissions = path.stat().st_mode + path.chmod(current_permissions | OWNER_WRITE_PERMISSION) class Command(ScrapyCommand): @@ -96,10 +95,7 @@ class Command(ScrapyCommand): project_name = args[0] - if len(args) == 2: - project_dir = Path(args[1]) - else: - project_dir = Path(args[0]) + project_dir = Path(args[-1]) if (project_dir / "scrapy.cfg").exists(): self.exitcode = 1 diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 3b4f932a0..bdb68c4ad 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -199,7 +199,7 @@ def _create_testcase(method: Callable, desc: str) -> TestCase: spider = method.__self__.name # type: ignore[attr-defined] class ContractTestCase(TestCase): - def __str__(_self) -> str: + def __str__(_self) -> str: # pylint: disable=no-self-argument return f"[{spider}] {method.__name__} ({desc})" name = f"{spider}_{method.__name__}" diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 598659b4d..0ad10baff 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -32,6 +32,7 @@ from __future__ import annotations import re from io import BytesIO +from pathlib import Path from typing import TYPE_CHECKING, Any, BinaryIO from urllib.parse import unquote @@ -56,9 +57,11 @@ if TYPE_CHECKING: class ReceivedDataProtocol(Protocol): - def __init__(self, filename: str | None = None): - self.__filename: str | None = filename - self.body: BinaryIO = open(filename, "wb") if filename else BytesIO() + def __init__(self, filename: bytes | None = None): + self.__filename: bytes | None = filename + self.body: BinaryIO = ( + Path(filename.decode()).open("wb") if filename else BytesIO() + ) self.size: int = 0 def dataReceived(self, data: bytes) -> None: @@ -66,7 +69,7 @@ class ReceivedDataProtocol(Protocol): self.size += len(data) @property - def filename(self) -> str | None: + def filename(self) -> bytes | None: return self.__filename def close(self) -> None: @@ -128,8 +131,8 @@ class FTPDownloadHandler: ) -> Response: self.result = result protocol.close() - headers = {"local filename": protocol.filename or "", "size": protocol.size} - body = to_bytes(protocol.filename or protocol.body.read()) + headers = {"local filename": protocol.filename or b"", "size": protocol.size} + body = protocol.filename or protocol.body.read() respcls = responsetypes.from_args(url=request.url, body=body) # hints for Headers-related types may need to be fixed to not use AnyStr return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type] diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9f65794fe..aa8a1a2a4 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -424,10 +424,7 @@ class ScrapyAgent: headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): headers.removeHeader(b"Proxy-Authorization") - if request.body: - bodyproducer = _RequestBodyProducer(request.body) - else: - bodyproducer = None + bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = time() d: Deferred[TxResponse] = agent.request( method, to_bytes(url, encoding="ascii"), headers, bodyproducer diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 5480df72c..61f444e31 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -291,9 +291,7 @@ class ExecutionEngine: return False if self.slot.start_requests is not None: # not all start requests are handled return False - if self.slot.scheduler.has_pending_requests(): - return False - return True + return not self.slot.scheduler.has_pending_requests() def crawl(self, request: Request) -> None: """Inject the request into the spider <-> downloader pipeline""" @@ -388,9 +386,8 @@ class ExecutionEngine: ) self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider - if hasattr(scheduler, "open"): - if d := scheduler.open(spider): - yield d + if hasattr(scheduler, "open") and (d := scheduler.open(spider)): + yield d yield self.scraper.open_spider(spider) assert self.crawler.stats self.crawler.stats.open_spider(spider) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index a63ee40bf..4b2520aa1 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -198,10 +198,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # chain, they went through it already from the process_spider_exception method recovered: MutableChain[_T] | MutableAsyncChain[_T] last_result_is_async = isinstance(result, AsyncIterable) - if last_result_is_async: - recovered = MutableAsyncChain() - else: - recovered = MutableChain() + recovered = MutableAsyncChain() if last_result_is_async else MutableChain() # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async. # 1. def foo. Sync iterables are passed as is, async ones are downgraded. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 05af1bf8a..0a28c4549 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib import logging import pprint import signal @@ -503,7 +504,6 @@ class CrawlerProcess(CrawlerRunner): def _stop_reactor(self, _: Any = None) -> None: from twisted.internet import reactor - try: + # raised if already stopped or in shutdown stage + with contextlib.suppress(RuntimeError): reactor.stop() - except RuntimeError: # raised if already stopped or in shutdown stage - pass diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index b74140ee1..80107261b 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -42,7 +42,10 @@ class HttpAuthMiddleware: self, request: Request, spider: Spider ) -> Request | Response | None: auth = getattr(self, "auth", None) - if auth and b"Authorization" not in request.headers: - if not self.domain or url_is_from_any_domain(request.url, [self.domain]): - request.headers[b"Authorization"] = auth + if ( + auth + and b"Authorization" not in request.headers + and (not self.domain or url_is_from_any_domain(request.url, [self.domain])) + ): + request.headers[b"Authorization"] = auth return None diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 2f3f2db47..cb7fa8c90 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -51,10 +51,7 @@ class HttpProxyMiddleware: proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", "")) - if user: - creds = self._basic_auth_header(user, password) - else: - creds = None + creds = self._basic_auth_header(user, password) if user else None return creds, proxy_url diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 0b883b43a..612426371 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -101,12 +101,14 @@ class BaseRedirectMiddleware: if ttl and redirects <= self.max_redirect_times: redirected.meta["redirect_times"] = redirects redirected.meta["redirect_ttl"] = ttl - 1 - redirected.meta["redirect_urls"] = request.meta.get("redirect_urls", []) + [ - request.url + redirected.meta["redirect_urls"] = [ + *request.meta.get("redirect_urls", []), + request.url, + ] + redirected.meta["redirect_reasons"] = [ + *request.meta.get("redirect_reasons", []), + reason, ] - redirected.meta["redirect_reasons"] = request.meta.get( - "redirect_reasons", [] - ) + [reason] redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust logger.debug( diff --git a/scrapy/exporters.py b/scrapy/exporters.py index cdb7ac159..46c6aa3fa 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -25,13 +25,13 @@ if TYPE_CHECKING: __all__ = [ "BaseItemExporter", - "PprintItemExporter", - "PickleItemExporter", "CsvItemExporter", - "XmlItemExporter", - "JsonLinesItemExporter", "JsonItemExporter", + "JsonLinesItemExporter", "MarshalItemExporter", + "PickleItemExporter", + "PprintItemExporter", + "XmlItemExporter", ] @@ -81,10 +81,7 @@ class BaseItemExporter: include_empty = self.export_empty_fields if self.fields_to_export is None: - if include_empty: - field_iter = item.field_names() - else: - field_iter = item.keys() + field_iter = item.field_names() if include_empty else item.keys() elif isinstance(self.fields_to_export, Mapping): if include_empty: field_iter = self.fields_to_export.items() diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 6948c394c..afaf81928 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst from __future__ import annotations +import contextlib import logging import signal import sys @@ -69,12 +70,10 @@ class StackTraceDump: class Debugger: def __init__(self) -> None: - try: + # win32 platforms don't support SIGUSR signals + with contextlib.suppress(AttributeError): signal.signal(signal.SIGUSR2, self._enter_debugger) # type: ignore[attr-defined] - except AttributeError: - # win32 platforms don't support SIGUSR signals - pass def _enter_debugger(self, signum: int, frame: FrameType | None) -> None: assert frame - Pdb().set_trace(frame.f_back) # noqa: T100 + Pdb().set_trace(frame.f_back) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index b6e6f55a6..c6e2aa0dd 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst from __future__ import annotations +import contextlib import logging import re import sys @@ -109,6 +110,8 @@ class ItemFilter: class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" + # pylint: disable=no-self-argument + def __init__(uri, *, feed_options=None): # pylint: disable=super-init-not-called """Initialize the storage with the parameters given in the URI and the feed-specific options (see :setting:`FEEDS`)""" @@ -642,10 +645,8 @@ class FeedExporter: ) d = {} for k, v in conf.items(): - try: + with contextlib.suppress(NotConfigured): d[k] = load_object(v) - except NotConfigured: - pass return d def _exporter_supported(self, format: str) -> bool: diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 929807de8..fe2cbcb86 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -89,10 +89,7 @@ class RFC2616Policy: return False cc = self._parse_cachecontrol(request) # obey user-agent directive "Cache-Control: no-store" - if b"no-store" in cc: - return False - # Any other is eligible for caching - return True + return b"no-store" not in cc def should_cache_response(self, response: Response, request: Request) -> bool: # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 7cf08a1bb..f97577442 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -151,10 +151,7 @@ class PeriodicLog: return False if exclude and not include: return True - for p in include: - if p in stat_name: - return True - return False + return any(p in stat_name for p in include) def spider_closed(self, spider: Spider, reason: str) -> None: self.log() diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index ee28d86ba..189b1953b 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -84,7 +84,9 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers - def login(self_, credentials, mind, *interfaces): + def login( + self_, credentials, mind, *interfaces + ): # pylint: disable=no-self-argument if not ( credentials.username == self.username.encode("utf8") and credentials.checkPassword(self.password.encode("utf8")) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 60322fe6e..b7c3b9d37 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -64,9 +64,8 @@ class CookieJar: cookies += self.jar._cookies_for_domain(host, wreq) # type: ignore[attr-defined] attrs = self.jar._cookie_attrs(cookies) # type: ignore[attr-defined] - if attrs: - if not wreq.has_header("Cookie"): - wreq.add_unredirected_header("Cookie", "; ".join(attrs)) + if attrs and not wreq.has_header("Cookie"): + wreq.add_unredirected_header("Cookie", "; ".join(attrs)) self.processed += 1 if self.processed % self.check_expired_frequency == 0: diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 289c60591..e26cbe05b 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: class JsonRequest(Request): - attributes: tuple[str, ...] = Request.attributes + ("dumps_kwargs",) + attributes: tuple[str, ...] = (*Request.attributes, "dumps_kwargs") def __init__( self, *args: Any, dumps_kwargs: dict[str, Any] | None = None, **kwargs: Any @@ -29,7 +29,7 @@ class JsonRequest(Request): dumps_kwargs.setdefault("sort_keys", True) self._dumps_kwargs: dict[str, Any] = dumps_kwargs - body_passed = kwargs.get("body", None) is not None + body_passed = kwargs.get("body") is not None data: Any = kwargs.pop("data", None) data_passed: bool = data is not None @@ -61,7 +61,7 @@ class JsonRequest(Request): def replace( self, *args: Any, cls: type[Request] | None = None, **kwargs: Any ) -> Request: - body_passed = kwargs.get("body", None) is not None + body_passed = kwargs.get("body") is not None data: Any = kwargs.pop("data", None) data_passed: bool = data is not None diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index f954b5e9e..476f1754e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -43,7 +43,7 @@ class TextResponse(Response): _DEFAULT_ENCODING = "ascii" _cached_decoded_json = _NONE - attributes: tuple[str, ...] = Response.attributes + ("encoding",) + attributes: tuple[str, ...] = (*Response.attributes, "encoding") def __init__(self, *args: Any, **kwargs: Any): self._encoding: str | None = kwargs.pop("encoding", None) diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index 9a2c5f170..b4f1d9394 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,3 +1,5 @@ +# pylint: disable=no-method-argument,no-self-argument + from zope.interface import Interface diff --git a/scrapy/link.py b/scrapy/link.py index 1a569f892..9c272ab2f 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -24,7 +24,7 @@ class Link: of the anchor tag. """ - __slots__ = ["url", "text", "fragment", "nofollow"] + __slots__ = ["fragment", "nofollow", "text", "url"] def __init__( self, url: str, text: str = "", fragment: str = "", nofollow: bool = False diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index f195dbdd7..4fd932b88 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -41,9 +41,12 @@ _collect_string_content = etree.XPath("string()") def _nons(tag: Any) -> Any: - if isinstance(tag, str): - if tag[0] == "{" and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE: - return tag.split("}")[-1] + if ( + isinstance(tag, str) + and tag[0] == "{" + and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE + ): + return tag.split("}")[-1] return tag @@ -230,9 +233,7 @@ class LxmlLinkExtractor: parsed_url, self.deny_extensions ): return False - if self.restrict_text and not _matches(link.text, self.restrict_text): - return False - return True + return not self.restrict_text or _matches(link.text, self.restrict_text) def matches(self, url: str) -> bool: if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 544f4adfe..76f9c7856 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -76,8 +76,8 @@ class LogFormatter: self, request: Request, response: Response, spider: Spider ) -> LogFormatterResult: """Logs a message when the crawler finds a webpage.""" - request_flags = f" {str(request.flags)}" if request.flags else "" - response_flags = f" {str(response.flags)}" if response.flags else "" + request_flags = f" {request.flags!s}" if request.flags else "" + response_flags = f" {response.flags!s}" if response.flags else "" return { "level": logging.DEBUG, "msg": CRAWLEDMSG, diff --git a/scrapy/mail.py b/scrapy/mail.py index a3c642401..be2423965 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -111,11 +111,9 @@ class MailSender: ) -> Deferred[None] | None: from twisted.internet import reactor - msg: MIMEBase - if attachs: - msg = MIMEMultipart() - else: - msg = MIMENonMultipart(*mimetype.split("/", 1)) + msg: MIMEBase = ( + MIMEMultipart() if attachs else MIMENonMultipart(*mimetype.split("/", 1)) + ) to = list(arg_to_iter(to)) cc = list(arg_to_iter(cc)) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 16bd45c00..a10117590 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -553,10 +553,8 @@ class FilesPipeline(MediaPipeline): ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") def _get_store(self, uri: str) -> FilesStoreProtocol: - if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir - scheme = "file" - else: - scheme = urlparse(uri).scheme + # to support win32 paths like: C:\\some\dir + scheme = "file" if Path(uri).is_absolute() else urlparse(uri).scheme store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 5438b8522..0f3329db1 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -127,8 +127,7 @@ class MediaPipeline(ABC): if ( not base_class_name or class_name == base_class_name - or settings - and not settings.get(formatted_key) + or (settings and not settings.get(formatted_key)) ): return key return formatted_key diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 5b2f81335..a04e0107b 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -34,7 +34,7 @@ def _path_safe(text: str) -> str: # as we replace some letters we can get collision for different slots # add we add unique part unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # noqa: S324 - return "-".join([pathable_slot, unique_slot]) + return f"{pathable_slot}-{unique_slot}" class QueueProtocol(Protocol): diff --git a/scrapy/shell.py b/scrapy/shell.py index 5d0ab1e4d..5e5e57a9a 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst from __future__ import annotations +import contextlib import os import signal from typing import TYPE_CHECKING, Any @@ -70,17 +71,16 @@ class Shell: else: self.populate_vars() if self.code: + # pylint: disable-next=eval-used print(eval(self.code, globals(), self.vars)) # noqa: S307 else: - """ - Detect interactive shell setting in scrapy.cfg - e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg - [settings] - # shell can be one of ipython, bpython or python; - # to be used as the interactive python console, if available. - # (default is ipython, fallbacks in the order listed above) - shell = python - """ + # Detect interactive shell setting in scrapy.cfg + # e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg + # [settings] + # # shell can be one of ipython, bpython or python; + # # to be used as the interactive python console, if available. + # # (default is ipython, fallbacks in the order listed above) + # shell = python cfg = get_config() section, option = "settings", "shell" env = os.environ.get("SCRAPY_PYTHON_SHELL") @@ -143,12 +143,10 @@ class Shell: else: request.meta["handle_httpstatus_all"] = True response = None - try: + with contextlib.suppress(IgnoreRequest): response, spider = threads.blockingCallFromThread( reactor, self._schedule, request, spider ) - except IgnoreRequest: - pass self.populate_vars(response, request, spider) def populate_vars( diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 93b7fcf17..a3a1e5b92 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -195,8 +195,7 @@ class StrictOriginPolicy(ReferrerPolicy): if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) - or not self.tls_protected(response_url) - ): + ) or not self.tls_protected(response_url): return self.origin_referrer(response_url) return None @@ -249,8 +248,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) - or not self.tls_protected(response_url) - ): + ) or not self.tls_protected(response_url): return self.origin_referrer(response_url) return None @@ -282,7 +280,7 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3") + NOREFERRER_SCHEMES: tuple[str, ...] = (*LOCAL_SCHEMES, "file", "s3") name: str = POLICY_SCRAPY_DEFAULT @@ -362,11 +360,10 @@ class RefererMiddleware: - otherwise, the policy from settings is used. """ policy_name = request.meta.get("referrer_policy") - if policy_name is None: - if isinstance(resp_or_url, Response): - policy_header = resp_or_url.headers.get("Referrer-Policy") - if policy_header is not None: - policy_name = to_unicode(policy_header.decode("latin1")) + if policy_name is None and isinstance(resp_or_url, Response): + policy_header = resp_or_url.headers.get("Referrer-Policy") + if policy_header is not None: + policy_name = to_unicode(policy_header.decode("latin1")) if policy_name is None: return self.default_policy() diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 591737b8e..6b09f36ff 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -1,3 +1,4 @@ +import contextlib import zlib from io import BytesIO from warnings import warn @@ -37,10 +38,8 @@ else: return decompressor.process(data) -try: +with contextlib.suppress(ImportError): import zstandard -except ImportError: - pass _CHUNK_SIZE = 65536 # 64 KiB diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 95844a48c..7425543ff 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -59,7 +59,7 @@ def _embed_ptpython_shell( namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a ptpython shell""" - import ptpython.repl + import ptpython.repl # pylint: disable=import-error @wraps(_embed_ptpython_shell) def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 98ecb2f02..3d0e0d3c7 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -8,6 +8,7 @@ This module must not depend on any module outside the Standard Library. from __future__ import annotations import collections +import contextlib import warnings import weakref from collections import OrderedDict @@ -173,10 +174,9 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): self.data: LocalCache = LocalCache(limit=limit) def __setitem__(self, key: _KT, value: _VT) -> None: - try: + # if raised, key is not weak-referenceable, skip caching + with contextlib.suppress(TypeError): super().__setitem__(key, value) - except TypeError: - pass # key is not weak-referenceable, skip caching def __getitem__(self, key: _KT) -> _VT | None: # type: ignore[override] try: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 0a0acc742..20d03cae6 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -57,6 +57,7 @@ def create_deprecated_class( # https://github.com/python/mypy/issues/4177 class DeprecatedClass(new_class.__class__): # type: ignore[misc, name-defined] + # pylint: disable=no-self-argument deprecated_class: type | None = None warned_on_subclass: bool = False diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 39f46270b..20744a604 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -30,6 +30,7 @@ def _tty_supports_color() -> bool: def _colorize(text: str, colorize: bool = True) -> str: + # pylint: disable=no-name-in-module if not colorize or not sys.stdout.isatty() or not _tty_supports_color(): return text try: diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 1948009e8..52f29e22c 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -32,7 +32,7 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]: checks: list[tuple[str, Any]] = [] for test in tests: try: - checks += [(test, eval(test))] # noqa: S307 + checks += [(test, eval(test))] # noqa: S307 # pylint: disable=eval-used except Exception as e: checks += [(test, f"{type(e).__name__} (exception)")] diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 5ce4863f6..d319e7950 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -252,7 +252,9 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: def returns_none(return_node: ast.Return) -> bool: value = return_node.value - return value is None or isinstance(value, ast.Constant) and value.value is None + return value is None or ( + isinstance(value, ast.Constant) and value.value is None + ) if inspect.isgeneratorfunction(callable): func = callable diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 66a06a9f0..679e38206 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -100,7 +100,7 @@ def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> No asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") - installer_path = module + ["install"] + installer_path = [*module, "install"] installer = load_object(".".join(installer_path)) with suppress(error.ReactorAlreadyInstalledError): installer() diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index ad811e804..7f2b178f5 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -229,7 +229,8 @@ def request_to_curl(request: Request) -> str: cookies = f"--cookie '{cookie}'" elif isinstance(request.cookies, list): cookie = "; ".join( - f"{list(c.keys())[0]}={list(c.values())[0]}" for c in request.cookies + f"{next(iter(c.keys()))}={next(iter(c.values()))}" + for c in request.cookies ) cookies = f"--cookie '{cookie}'" diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index c1d3bfffb..5fd176a3f 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -36,7 +36,7 @@ def send_catch_log( dont_log = named.pop("dont_log", ()) dont_log = tuple(dont_log) if isinstance(dont_log, Sequence) else (dont_log,) dont_log += (StopDownload,) - spider = named.get("spider", None) + spider = named.get("spider") responses: list[tuple[TypingAny, TypingAny]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): result: TypingAny @@ -88,7 +88,7 @@ def send_catch_log_deferred( return failure dont_log = named.pop("dont_log", None) - spider = named.get("spider", None) + spider = named.get("spider") dfds: list[Deferred[tuple[TypingAny, TypingAny]]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): d: Deferred[TypingAny] = maybeDeferred_coro( diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 05e04e2d1..85d7c940f 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: class ProcessTest: command: str | None = None prefix = [sys.executable, "-m", "scrapy.cmdline"] - cwd = os.getcwd() # trial chdirs to temp dir + cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109 def execute( self, @@ -31,7 +31,7 @@ class ProcessTest: if settings is not None: env["SCRAPY_SETTINGS_MODULE"] = settings assert self.command - cmd = self.prefix + [self.command] + list(args) + cmd = [*self.prefix, self.command, *args] pp = TestProcessProtocol() pp.deferred.addCallback(self._process_finished, cmd, check_code) reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 3bf831c26..db2749d79 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -51,7 +51,7 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool: """Return True if the url belongs to the given spider""" return url_is_from_any_domain( - url, [spider.name] + list(getattr(spider, "allowed_domains", [])) + url, [spider.name, *getattr(spider, "allowed_domains", [])] ) @@ -173,13 +173,19 @@ def strip_url( parsed_url.username or parsed_url.password ): netloc = netloc.split("@")[-1] - if strip_default_port and parsed_url.port: - if (parsed_url.scheme, parsed_url.port) in ( + + if ( + strip_default_port + and parsed_url.port + and (parsed_url.scheme, parsed_url.port) + in ( ("http", 80), ("https", 443), ("ftp", 21), - ): - netloc = netloc.replace(f":{parsed_url.port}", "") + ) + ): + netloc = netloc.replace(f":{parsed_url.port}", "") + return urlunparse( ( parsed_url.scheme, diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index ff1f9b346..052321ae3 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -11,7 +11,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings.default_settings import LOG_VERSIONS from scrapy.utils.ssl import get_openssl_version -_DEFAULT_SOFTWARE = ["Scrapy"] + LOG_VERSIONS +_DEFAULT_SOFTWARE = ["Scrapy", *LOG_VERSIONS] def _version(item): diff --git a/tests/test_addons.py b/tests/test_addons.py index 17949997c..a0caa3511 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -166,19 +166,21 @@ class AddonManagerTest(unittest.TestCase): def update_settings(self, settings): pass - with patch("scrapy.addons.logger") as logger_mock: - with patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock: - settings_dict = { - "ADDONS": {LoggedAddon: 1}, - } - addon = LoggedAddon() - build_from_crawler_mock.return_value = addon - crawler = get_crawler(settings_dict=settings_dict) - logger_mock.info.assert_called_once_with( - "Enabled addons:\n%(addons)s", - {"addons": [addon]}, - extra={"crawler": crawler}, - ) + with ( + patch("scrapy.addons.logger") as logger_mock, + patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock, + ): + settings_dict = { + "ADDONS": {LoggedAddon: 1}, + } + addon = LoggedAddon() + build_from_crawler_mock.return_value = addon + crawler = get_crawler(settings_dict=settings_dict) + logger_mock.info.assert_called_once_with( + "Enabled addons:\n%(addons)s", + {"addons": [addon]}, + extra={"crawler": crawler}, + ) @inlineCallbacks def test_enable_addon_in_spider(self): diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 4835e936b..acd524ea4 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -21,7 +21,7 @@ class CmdlineTest(unittest.TestCase): def _execute(self, *new_args, **kwargs): encoding = sys.stdout.encoding or "utf-8" - args = (sys.executable, "-m", "scrapy.cmdline") + new_args + args = (sys.executable, "-m", "scrapy.cmdline", *new_args) proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs) comm = proc.communicate()[0].strip() return comm.decode(encoding) diff --git a/tests/test_commands.py b/tests/test_commands.py index 32b69de8a..9d5720b98 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -87,13 +87,13 @@ class ProjectTest(unittest.TestCase): def call(self, *new_args, **kwargs): with TemporaryFile() as out: - args = (sys.executable, "-m", "scrapy.cmdline") + new_args + args = (sys.executable, "-m", "scrapy.cmdline", *new_args) return subprocess.call( args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs ) def proc(self, *new_args, **popen_kwargs): - args = (sys.executable, "-m", "scrapy.cmdline") + new_args + args = (sys.executable, "-m", "scrapy.cmdline", *new_args) p = subprocess.Popen( args, cwd=popen_kwargs.pop("cwd", self.cwd), diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 743889234..f7581707b 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -517,8 +517,8 @@ class ContractsManagerTest(unittest.TestCase): super().__init__(*args, **kwargs) self.visited = 0 - def start_requests(s): - return self.conman.from_spider(s, self.results) + def start_requests(self_): # pylint: disable=no-self-argument + return self.conman.from_spider(self_, self.results) def parse_first(self, response): self.visited += 1 diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f3e5ebf5d..6c3fe96b0 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,5 +1,4 @@ import logging -import os import platform import re import signal @@ -643,11 +642,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: script_dir: Path - cwd = os.getcwd() def get_script_args(self, script_name: str, *script_args: str) -> list[str]: script_path = self.script_dir / script_name - return [sys.executable, str(script_path)] + list(script_args) + return [sys.executable, str(script_path), *script_args] def run_script(self, script_name: str, *script_args: str) -> str: args = self.get_script_args(script_name, *script_args) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8ecba41bf..0dcbeaec1 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -116,7 +116,7 @@ class FileTestCase(unittest.TestCase): def tearDown(self): os.close(self.fd) - os.remove(self.tmpname) + Path(self.tmpname).unlink() def test_download(self): def _test(response): @@ -530,9 +530,10 @@ class Http11TestCase(HttpTestCase): d = self.download_request(request, Spider("foo")) def checkDataLoss(failure): - if failure.check(ResponseFailed): - if any(r.check(_DataLoss) for r in failure.value.reasons): - return None + if failure.check(ResponseFailed) and any( + r.check(_DataLoss) for r in failure.value.reasons + ): + return None return failure d.addCallback(lambda _: self.fail("No DataLoss exception")) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 9b39b84d9..6b9b39413 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -114,7 +114,7 @@ class RetryTest(unittest.TestCase): def test_exception_to_retry_added(self): exc = ValueError settings_dict = { - "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc], + "RETRY_EXCEPTIONS": [*RETRY_EXCEPTIONS, exc], } crawler = get_crawler(Spider, settings_dict=settings_dict) mw = RetryMiddleware.from_crawler(crawler) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 55f9ecac9..879bc8697 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -31,7 +31,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): def start_requests(self): self.times = {None: []} - slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None] + slots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None] for slot in slots: url = self.mockserver.url(f"/?downloader_slot={slot}") diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 0f70887af..1fbacfdfc 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -116,7 +116,7 @@ class BaseItemExporterTest(unittest.TestCase): ) ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1") - _, name = list(ie._get_serialized_fields(self.i))[0] + _, name = next(iter(ie._get_serialized_fields(self.i))) assert isinstance(name, str) self.assertEqual(name, "John\xa3") @@ -216,7 +216,9 @@ class PprintItemExporterTest(BaseItemExporterTest): return PprintItemExporter(self.output, **kwargs) def _check_output(self): - self._assert_expected_item(eval(self.output.getvalue())) + self._assert_expected_item( + eval(self.output.getvalue()) # pylint: disable=eval-used + ) class PprintItemExporterDataclassTest(PprintItemExporterTest): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b087aaab1..0f149f172 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -756,7 +756,7 @@ class FeedExportTest(FeedExportTestBase): ) finally: - for file_path in FEEDS.keys(): + for file_path in FEEDS: if not Path(file_path).exists(): continue @@ -1229,15 +1229,13 @@ class FeedExportTest(FeedExportTestBase): class CustomFilter2(scrapy.extensions.feedexport.ItemFilter): def accepts(self, item): - if "foo" not in item.fields: - return False - return True + return "foo" in item.fields class CustomFilter3(scrapy.extensions.feedexport.ItemFilter): def accepts(self, item): - if isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1": - return True - return False + return ( + isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1" + ) formats = { "json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]', diff --git a/tests/test_http_request.py b/tests/test_http_request.py index d020a8911..34d3b25d5 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1488,10 +1488,7 @@ def _buildresponse(body, **kwargs): def _qs(req, encoding="utf-8", to_unicode=False): - if req.method == "POST": - qs = req.body - else: - qs = req.url.partition("?")[2] + qs = req.body if req.method == "POST" else req.url.partition("?")[2] uqs = unquote_to_bytes(qs) if to_unicode: uqs = uqs.decode(encoding) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 679cc8238..0730cff3a 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -960,7 +960,7 @@ class XmlResponseTest(TextResponseTest): class CustomResponse(TextResponse): - attributes = TextResponse.attributes + ("foo", "bar") + attributes = (*TextResponse.attributes, "foo", "bar") def __init__(self, *args, **kwargs) -> None: self.foo = kwargs.pop("foo", None) diff --git a/tests/test_item.py b/tests/test_item.py index 3f10a724d..5a8ee095e 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -54,7 +54,7 @@ class ItemTest(unittest.TestCase): self.assertEqual(itemrepr, "{'name': 'John Doe', 'number': 123}") - i2 = eval(itemrepr) + i2 = eval(itemrepr) # pylint: disable=eval-used self.assertEqual(i2["name"], "John Doe") self.assertEqual(i2["number"], 123) diff --git a/tests/test_link.py b/tests/test_link.py index 7ba0851ae..35723bbd6 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -49,7 +49,7 @@ class LinkTest(unittest.TestCase): l1 = Link( "http://www.example.com", text="test", fragment="something", nofollow=True ) - l2 = eval(repr(l1)) + l2 = eval(repr(l1)) # pylint: disable=eval-used self._assert_same_links(l1, l2) def test_bytes_url(self): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index a6c5f0a94..4c3fc36b6 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -634,19 +634,21 @@ class TestGCSFilesStore(unittest.TestCase): import google.cloud.storage # noqa: F401 except ModuleNotFoundError: raise unittest.SkipTest("google-cloud-storage is not installed") - with mock.patch("google.cloud.storage") as _: - with mock.patch("scrapy.pipelines.files.time") as _: - uri = "gs://my_bucket/my_prefix/" - store = GCSFilesStore(uri) - store.bucket = mock.Mock() - path = "full/my_data.txt" - yield store.persist_file( - path, mock.Mock(), info=None, meta=None, headers=None - ) - yield store.stat_file(path, info=None) - expected_blob_path = store.prefix + path - store.bucket.blob.assert_called_with(expected_blob_path) - store.bucket.get_blob.assert_called_with(expected_blob_path) + with ( + mock.patch("google.cloud.storage"), + mock.patch("scrapy.pipelines.files.time"), + ): + uri = "gs://my_bucket/my_prefix/" + store = GCSFilesStore(uri) + store.bucket = mock.Mock() + path = "full/my_data.txt" + yield store.persist_file( + path, mock.Mock(), info=None, meta=None, headers=None + ) + yield store.stat_file(path, info=None) + expected_blob_path = store.prefix + path + store.bucket.blob.assert_called_with(expected_blob_path) + store.bucket.get_blob.assert_called_with(expected_blob_path) class TestFTPFileStore(unittest.TestCase): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 8bd1480ad..3ac330ae2 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -277,14 +277,12 @@ class DownloaderAwareSchedulerTestMixin: downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() - # pylint: disable=protected-access slot = downloader.get_slot_key(request) dequeued_slots.append(slot) downloader.increment(slot) requests.append(request) for request in requests: - # pylint: disable=protected-access slot = downloader.get_slot_key(request) downloader.decrement(slot) diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.py index 979c42267..443e26a3c 100644 --- a/tests/test_scrapy__getattr__.py +++ b/tests/test_scrapy__getattr__.py @@ -3,7 +3,7 @@ import warnings def test_deprecated_twisted_version(): with warnings.catch_warnings(record=True) as warns: - from scrapy import twisted_version + from scrapy import twisted_version # pylint: disable=no-name-in-module assert twisted_version is not None assert isinstance(twisted_version, tuple) diff --git a/tests/test_selector.py b/tests/test_selector.py index 1b5f3f018..857c7d626 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -264,7 +264,7 @@ class JMESPathTestCase(unittest.TestCase): ) @pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath") - def test_jmespath_not_available(my_json_page) -> None: + def test_jmespath_not_available(self) -> None: body = """ { "website": {"name": "Example"} diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 8bc48aa7b..5c8a19d9b 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -170,7 +170,7 @@ class BaseSettingsTest(unittest.TestCase): self.assertCountEqual(self.settings.attributes.keys(), ctrl_attributes.keys()) - for key in ctrl_attributes.keys(): + for key in ctrl_attributes: attr = self.settings.attributes[key] ctrl_attr = ctrl_attributes[key] self.assertEqual(attr.value, ctrl_attr.value) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 9b53b9b96..d5aac34eb 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,3 +1,4 @@ +import contextlib import shutil import sys import tempfile @@ -22,10 +23,8 @@ module_dir = Path(__file__).resolve().parent def _copytree(source: Path, target: Path): - try: + with contextlib.suppress(shutil.Error): shutil.copytree(source, target) - except shutil.Error: - pass class SpiderLoaderTest(unittest.TestCase): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 23b0c17c6..4945ac25d 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -891,13 +891,11 @@ class TestSettingsPolicyByName(TestCase): # test parsing without space(s) after the comma settings1 = Settings( { - "REFERRER_POLICY": ",".join( - [ - "some-custom-unknown-policy", - POLICY_SAME_ORIGIN, - POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, - "another-custom-unknown-policy", - ] + "REFERRER_POLICY": ( + f"some-custom-unknown-policy," + f"{POLICY_SAME_ORIGIN}," + f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}," + f"another-custom-unknown-policy" ) } ) @@ -907,12 +905,10 @@ class TestSettingsPolicyByName(TestCase): # test parsing with space(s) after the comma settings2 = Settings( { - "REFERRER_POLICY": ", ".join( - [ - POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, - "another-custom-unknown-policy", - POLICY_UNSAFE_URL, - ] + "REFERRER_POLICY": ( + f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}," + f" another-custom-unknown-policy," + f" {POLICY_UNSAFE_URL}" ) } ) @@ -922,12 +918,10 @@ class TestSettingsPolicyByName(TestCase): def test_multiple_policy_tokens_all_invalid(self): settings = Settings( { - "REFERRER_POLICY": ",".join( - [ - "some-custom-unknown-policy", - "another-custom-unknown-policy", - "yet-another-custom-unknown-policy", - ] + "REFERRER_POLICY": ( + "some-custom-unknown-policy," + "another-custom-unknown-policy," + "yet-another-custom-unknown-policy" ) } ) diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 02ea8027f..04eeae4dc 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -1,3 +1,7 @@ +""" +Queues that handle requests +""" + import shutil import tempfile import unittest @@ -16,10 +20,6 @@ from scrapy.squeues import ( ) from scrapy.utils.test import get_crawler -""" -Queues that handle requests -""" - class BaseQueueTestCase(unittest.TestCase): def setUp(self): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index eedb6f6af..dc5fbd3c3 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -259,12 +259,14 @@ class WarnWhenSubclassedTest(unittest.TestCase): self.assertIn("foo.Bar", str(w[1].message)) def test_inspect_stack(self): - with mock.patch("inspect.stack", side_effect=IndexError): - with warnings.catch_warnings(record=True) as w: - DeprecatedName = create_deprecated_class("DeprecatedName", NewName) + with ( + mock.patch("inspect.stack", side_effect=IndexError), + warnings.catch_warnings(record=True) as w, + ): + DeprecatedName = create_deprecated_class("DeprecatedName", NewName) - class SubClass(DeprecatedName): - pass + class SubClass(DeprecatedName): + pass self.assertIn("Error detecting parent module", str(w[0].message)) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 4c81e3a2f..12507c6a3 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -366,7 +366,7 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all(isinstance(k, str) for k in result_row.keys())) + self.assertTrue(all(isinstance(k, str) for k in result_row)) self.assertTrue(all(isinstance(v, str) for v in result_row.values())) def test_csviter_delimiter(self): diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 3831f4c21..1d149d48d 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -12,7 +12,7 @@ from scrapy.utils.project import data_path, get_project_settings @contextlib.contextmanager def inside_a_project(): - prev_dir = os.getcwd() + prev_dir = Path.cwd() project_dir = tempfile.mkdtemp() try: diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index ef07d625f..58efad585 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -61,11 +61,11 @@ Foo 1 oldest: 0s ago\n\n""", ) def test_get_oldest(self): - o1 = Foo() # noqa: F841 + o1 = Foo() o1_time = time() - o2 = Bar() # noqa: F841 + o2 = Bar() o3_time = time() if o3_time <= o1_time: @@ -80,9 +80,9 @@ Foo 1 oldest: 0s ago\n\n""", self.assertIsNone(trackref.get_oldest("XXX")) def test_iter_all(self): - o1 = Foo() # noqa: F841 + o1 = Foo() o2 = Bar() # noqa: F841 - o3 = Foo() # noqa: F841 + o3 = Foo() self.assertEqual( set(trackref.iter_all("Foo")), {o1, o3},