diff --git a/pyproject.toml b/pyproject.toml index ad85e5c75..065382205 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -246,6 +246,8 @@ extend-select = [ "RET", # flake8-raise "RSE", + # Ruff-specific rules + "RUF", # flake8-bandit "S", # flake8-slots @@ -324,6 +326,14 @@ ignore = [ "PLR2004", # `for` loop variable overwritten by assignment target "PLW2901", + # String contains ambiguous {}. + "RUF001", + # Docstring contains ambiguous {}. + "RUF002", + # Comment contains ambiguous {}. + "RUF003", + # Mutable class attributes should be annotated with `typing.ClassVar` + "RUF012", # Use of `assert` detected; needed for mypy "S101", # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index c19710a6a..256504c9c 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -13,14 +13,14 @@ from scrapy.selector import Selector from scrapy.spiders import Spider __all__ = [ + "Field", + "FormRequest", + "Item", + "Request", + "Selector", + "Spider", "__version__", "version_info", - "Spider", - "Request", - "FormRequest", - "Selector", - "Item", - "Field", ] diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 86d4cc41c..184bd5ca4 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -39,9 +39,8 @@ class Command(BaseRunSpiderCommand): else: self.crawler_process.start() - if ( - self.crawler_process.bootstrap_failed - or hasattr(self.crawler_process, "has_exception") + if self.crawler_process.bootstrap_failed or ( + hasattr(self.crawler_process, "has_exception") and self.crawler_process.has_exception ): self.exitcode = 1 diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index cc5c1350b..f996d1806 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -269,7 +269,7 @@ class Command(BaseRunSpiderCommand): assert self.crawler_process assert self.spidercls self.crawler_process.crawl(self.spidercls, **opts.spargs) - self.pcrawler = list(self.crawler_process.crawlers)[0] + self.pcrawler = next(iter(self.crawler_process.crawlers)) self.crawler_process.start() if not self.first_response: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index bf8e41020..357ca8b37 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -20,7 +20,7 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType: if abspath.suffix not in (".py", ".pyw"): raise ValueError(f"Not a Python source file: {abspath}") dirname = str(abspath.parent) - sys.path = [dirname] + sys.path + sys.path = [dirname, *sys.path] try: module = import_module(abspath.stem) finally: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 0b883b43a..612426371 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -101,12 +101,14 @@ class BaseRedirectMiddleware: if ttl and redirects <= self.max_redirect_times: redirected.meta["redirect_times"] = redirects redirected.meta["redirect_ttl"] = ttl - 1 - redirected.meta["redirect_urls"] = request.meta.get("redirect_urls", []) + [ - request.url + redirected.meta["redirect_urls"] = [ + *request.meta.get("redirect_urls", []), + request.url, + ] + redirected.meta["redirect_reasons"] = [ + *request.meta.get("redirect_reasons", []), + reason, ] - redirected.meta["redirect_reasons"] = request.meta.get( - "redirect_reasons", [] - ) + [reason] redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust logger.debug( diff --git a/scrapy/exporters.py b/scrapy/exporters.py index cdb7ac159..834a05ae9 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -25,13 +25,13 @@ if TYPE_CHECKING: __all__ = [ "BaseItemExporter", - "PprintItemExporter", - "PickleItemExporter", "CsvItemExporter", - "XmlItemExporter", - "JsonLinesItemExporter", "JsonItemExporter", + "JsonLinesItemExporter", "MarshalItemExporter", + "PickleItemExporter", + "PprintItemExporter", + "XmlItemExporter", ] diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 6948c394c..5ca07394f 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -77,4 +77,4 @@ class Debugger: def _enter_debugger(self, signum: int, frame: FrameType | None) -> None: assert frame - Pdb().set_trace(frame.f_back) # noqa: T100 + Pdb().set_trace(frame.f_back) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 289c60591..e5b63ef14 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: class JsonRequest(Request): - attributes: tuple[str, ...] = Request.attributes + ("dumps_kwargs",) + attributes: tuple[str, ...] = (*Request.attributes, "dumps_kwargs") def __init__( self, *args: Any, dumps_kwargs: dict[str, Any] | None = None, **kwargs: Any diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index f954b5e9e..476f1754e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -43,7 +43,7 @@ class TextResponse(Response): _DEFAULT_ENCODING = "ascii" _cached_decoded_json = _NONE - attributes: tuple[str, ...] = Response.attributes + ("encoding",) + attributes: tuple[str, ...] = (*Response.attributes, "encoding") def __init__(self, *args: Any, **kwargs: Any): self._encoding: str | None = kwargs.pop("encoding", None) diff --git a/scrapy/link.py b/scrapy/link.py index 1a569f892..9c272ab2f 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -24,7 +24,7 @@ class Link: of the anchor tag. """ - __slots__ = ["url", "text", "fragment", "nofollow"] + __slots__ = ["fragment", "nofollow", "text", "url"] def __init__( self, url: str, text: str = "", fragment: str = "", nofollow: bool = False diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 544f4adfe..76f9c7856 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -76,8 +76,8 @@ class LogFormatter: self, request: Request, response: Response, spider: Spider ) -> LogFormatterResult: """Logs a message when the crawler finds a webpage.""" - request_flags = f" {str(request.flags)}" if request.flags else "" - response_flags = f" {str(response.flags)}" if response.flags else "" + request_flags = f" {request.flags!s}" if request.flags else "" + response_flags = f" {response.flags!s}" if response.flags else "" return { "level": logging.DEBUG, "msg": CRAWLEDMSG, diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 5438b8522..0f3329db1 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -127,8 +127,7 @@ class MediaPipeline(ABC): if ( not base_class_name or class_name == base_class_name - or settings - and not settings.get(formatted_key) + or (settings and not settings.get(formatted_key)) ): return key return formatted_key diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 93b7fcf17..18cc991bf 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -195,8 +195,7 @@ class StrictOriginPolicy(ReferrerPolicy): if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) - or not self.tls_protected(response_url) - ): + ) or not self.tls_protected(response_url): return self.origin_referrer(response_url) return None @@ -249,8 +248,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) - or not self.tls_protected(response_url) - ): + ) or not self.tls_protected(response_url): return self.origin_referrer(response_url) return None @@ -282,7 +280,7 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3") + NOREFERRER_SCHEMES: tuple[str, ...] = (*LOCAL_SCHEMES, "file", "s3") name: str = POLICY_SCRAPY_DEFAULT diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 5ce4863f6..d319e7950 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -252,7 +252,9 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: def returns_none(return_node: ast.Return) -> bool: value = return_node.value - return value is None or isinstance(value, ast.Constant) and value.value is None + return value is None or ( + isinstance(value, ast.Constant) and value.value is None + ) if inspect.isgeneratorfunction(callable): func = callable diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 66a06a9f0..679e38206 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -100,7 +100,7 @@ def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> No asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") - installer_path = module + ["install"] + installer_path = [*module, "install"] installer = load_object(".".join(installer_path)) with suppress(error.ReactorAlreadyInstalledError): installer() diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index ad811e804..7f2b178f5 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -229,7 +229,8 @@ def request_to_curl(request: Request) -> str: cookies = f"--cookie '{cookie}'" elif isinstance(request.cookies, list): cookie = "; ".join( - f"{list(c.keys())[0]}={list(c.values())[0]}" for c in request.cookies + f"{next(iter(c.keys()))}={next(iter(c.values()))}" + for c in request.cookies ) cookies = f"--cookie '{cookie}'" diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 05e04e2d1..3b1035eab 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -31,7 +31,7 @@ class ProcessTest: if settings is not None: env["SCRAPY_SETTINGS_MODULE"] = settings assert self.command - cmd = self.prefix + [self.command] + list(args) + cmd = [*self.prefix, self.command, *args] pp = TestProcessProtocol() pp.deferred.addCallback(self._process_finished, cmd, check_code) reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 3bf831c26..d487849bb 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -51,7 +51,7 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool: """Return True if the url belongs to the given spider""" return url_is_from_any_domain( - url, [spider.name] + list(getattr(spider, "allowed_domains", [])) + url, [spider.name, *getattr(spider, "allowed_domains", [])] ) diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index ff1f9b346..052321ae3 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -11,7 +11,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings.default_settings import LOG_VERSIONS from scrapy.utils.ssl import get_openssl_version -_DEFAULT_SOFTWARE = ["Scrapy"] + LOG_VERSIONS +_DEFAULT_SOFTWARE = ["Scrapy", *LOG_VERSIONS] def _version(item): diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 4835e936b..acd524ea4 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -21,7 +21,7 @@ class CmdlineTest(unittest.TestCase): def _execute(self, *new_args, **kwargs): encoding = sys.stdout.encoding or "utf-8" - args = (sys.executable, "-m", "scrapy.cmdline") + new_args + args = (sys.executable, "-m", "scrapy.cmdline", *new_args) proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs) comm = proc.communicate()[0].strip() return comm.decode(encoding) diff --git a/tests/test_commands.py b/tests/test_commands.py index 32b69de8a..9d5720b98 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -87,13 +87,13 @@ class ProjectTest(unittest.TestCase): def call(self, *new_args, **kwargs): with TemporaryFile() as out: - args = (sys.executable, "-m", "scrapy.cmdline") + new_args + args = (sys.executable, "-m", "scrapy.cmdline", *new_args) return subprocess.call( args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs ) def proc(self, *new_args, **popen_kwargs): - args = (sys.executable, "-m", "scrapy.cmdline") + new_args + args = (sys.executable, "-m", "scrapy.cmdline", *new_args) p = subprocess.Popen( args, cwd=popen_kwargs.pop("cwd", self.cwd), diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f3e5ebf5d..8b3a6eeca 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -647,7 +647,7 @@ class ScriptRunnerMixin: def get_script_args(self, script_name: str, *script_args: str) -> list[str]: script_path = self.script_dir / script_name - return [sys.executable, str(script_path)] + list(script_args) + return [sys.executable, str(script_path), *script_args] def run_script(self, script_name: str, *script_args: str) -> str: args = self.get_script_args(script_name, *script_args) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index c99f19b03..1eb7dcf9d 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -114,7 +114,7 @@ class RetryTest(unittest.TestCase): def test_exception_to_retry_added(self): exc = ValueError settings_dict = { - "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc], + "RETRY_EXCEPTIONS": [*RETRY_EXCEPTIONS, exc], } crawler = get_crawler(Spider, settings_dict=settings_dict) mw = RetryMiddleware.from_crawler(crawler) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 55f9ecac9..879bc8697 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -31,7 +31,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): def start_requests(self): self.times = {None: []} - slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None] + slots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None] for slot in slots: url = self.mockserver.url(f"/?downloader_slot={slot}") diff --git a/tests/test_exporters.py b/tests/test_exporters.py index fa9389044..522c6638d 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -116,7 +116,7 @@ class BaseItemExporterTest(unittest.TestCase): ) ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1") - _, name = list(ie._get_serialized_fields(self.i))[0] + _, name = next(iter(ie._get_serialized_fields(self.i))) assert isinstance(name, str) self.assertEqual(name, "John\xa3") diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 679cc8238..0730cff3a 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -960,7 +960,7 @@ class XmlResponseTest(TextResponseTest): class CustomResponse(TextResponse): - attributes = TextResponse.attributes + ("foo", "bar") + attributes = (*TextResponse.attributes, "foo", "bar") def __init__(self, *args, **kwargs) -> None: self.foo = kwargs.pop("foo", None) diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index ef07d625f..58efad585 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -61,11 +61,11 @@ Foo 1 oldest: 0s ago\n\n""", ) def test_get_oldest(self): - o1 = Foo() # noqa: F841 + o1 = Foo() o1_time = time() - o2 = Bar() # noqa: F841 + o2 = Bar() o3_time = time() if o3_time <= o1_time: @@ -80,9 +80,9 @@ Foo 1 oldest: 0s ago\n\n""", self.assertIsNone(trackref.get_oldest("XXX")) def test_iter_all(self): - o1 = Foo() # noqa: F841 + o1 = Foo() o2 = Bar() # noqa: F841 - o3 = Foo() # noqa: F841 + o3 = Foo() self.assertEqual( set(trackref.iter_all("Foo")), {o1, o3},