diff --git a/pyproject.toml b/pyproject.toml index a0b37b966..08b4b09b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -240,6 +240,8 @@ extend-select = [ "PIE", # pylint "PL", + # flake8-use-pathlib + "PTH", # flake8-pyi "PYI", # flake8-quotes diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index d7dc104c2..2a1dea997 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -154,7 +154,7 @@ class Command(ScrapyCommand): spiders_dir = Path(spiders_module.__file__).parent.resolve() else: spiders_module = None - spiders_dir = Path(".") + spiders_dir = Path() spider_file = f"{spiders_dir / module}.py" shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 5cb73f0d2..e0c004580 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import re import string from importlib.util import find_spec @@ -28,9 +27,9 @@ TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") -def _make_writable(path: str | os.PathLike) -> None: - current_permissions = os.stat(path).st_mode - os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION) +def _make_writable(path: Path) -> None: + current_permissions = path.stat().st_mode + path.chmod(current_permissions | OWNER_WRITE_PERMISSION) class Command(ScrapyCommand): diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 598659b4d..0ad10baff 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -32,6 +32,7 @@ from __future__ import annotations import re from io import BytesIO +from pathlib import Path from typing import TYPE_CHECKING, Any, BinaryIO from urllib.parse import unquote @@ -56,9 +57,11 @@ if TYPE_CHECKING: class ReceivedDataProtocol(Protocol): - def __init__(self, filename: str | None = None): - self.__filename: str | None = filename - self.body: BinaryIO = open(filename, "wb") if filename else BytesIO() + def __init__(self, filename: bytes | None = None): + self.__filename: bytes | None = filename + self.body: BinaryIO = ( + Path(filename.decode()).open("wb") if filename else BytesIO() + ) self.size: int = 0 def dataReceived(self, data: bytes) -> None: @@ -66,7 +69,7 @@ class ReceivedDataProtocol(Protocol): self.size += len(data) @property - def filename(self) -> str | None: + def filename(self) -> bytes | None: return self.__filename def close(self) -> None: @@ -128,8 +131,8 @@ class FTPDownloadHandler: ) -> Response: self.result = result protocol.close() - headers = {"local filename": protocol.filename or "", "size": protocol.size} - body = to_bytes(protocol.filename or protocol.body.read()) + headers = {"local filename": protocol.filename or b"", "size": protocol.size} + body = protocol.filename or protocol.body.read() respcls = responsetypes.from_args(url=request.url, body=body) # hints for Headers-related types may need to be fixed to not use AnyStr return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type] diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 3b1035eab..85d7c940f 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: class ProcessTest: command: str | None = None prefix = [sys.executable, "-m", "scrapy.cmdline"] - cwd = os.getcwd() # trial chdirs to temp dir + cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109 def execute( self, diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 8b3a6eeca..6c3fe96b0 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,5 +1,4 @@ import logging -import os import platform import re import signal @@ -643,7 +642,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: script_dir: Path - cwd = os.getcwd() def get_script_args(self, script_name: str, *script_args: str) -> list[str]: script_path = self.script_dir / script_name diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8ecba41bf..05b64e704 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -116,7 +116,7 @@ class FileTestCase(unittest.TestCase): def tearDown(self): os.close(self.fd) - os.remove(self.tmpname) + Path(self.tmpname).unlink() def test_download(self): def _test(response): diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 3831f4c21..1d149d48d 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -12,7 +12,7 @@ from scrapy.utils.project import data_path, get_project_settings @contextlib.contextmanager def inside_a_project(): - prev_dir = os.getcwd() + prev_dir = Path.cwd() project_dir = tempfile.mkdtemp() try: