diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 49db3f610..b273e269b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,12 +3,6 @@ repos: rev: v0.8.1 hooks: - id: ruff -- repo: https://github.com/PyCQA/bandit - rev: 1.7.9 - hooks: - - id: bandit - args: ["-c", "pyproject.toml"] - additional_dependencies: ["bandit[toml]"] - repo: https://github.com/psf/black.git rev: 24.4.2 hooks: @@ -23,8 +17,3 @@ repos: - id: blacken-docs additional_dependencies: - black==24.4.2 -- repo: https://github.com/asottile/pyupgrade - rev: v3.18.0 - hooks: - - id: pyupgrade - args: [--py39-plus] diff --git a/pyproject.toml b/pyproject.toml index 1378bab50..977792178 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,16 +92,6 @@ follow_imports = "skip" module = "scrapy.settings.default_settings" ignore_errors = true -[tool.bandit] -skips = [ - "B101", # assert_used, needed for mypy - "B321", # ftplib, https://github.com/scrapy/scrapy/issues/4180 - "B402", # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 - "B411", # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 - "B503", # ssl_with_bad_defaults -] -exclude_dirs = ["tests"] - [tool.bumpversion] current_version = "2.12.0" commit = true @@ -242,10 +232,30 @@ extend-select = [ "C4", # pydocstyle "D", + # flake8-future-annotations + "FA", + # refurb + "FURB", + # flake8-implicit-str-concat + "ISC", + # flake8-logging + "LOG", + # pygrep-hooks + "PGH", + # flake8-quotes + "Q", + # flake8-bandit + "S", + # flake8-slots + "SLOT", # flake8-debugger "T10", # flake8-type-checking "TC", + # pyupgrade + "UP", + # flake8-2020 + "YTT", ] ignore = [ # Assigning to `os.environ` doesn't clear the environment. @@ -296,6 +306,12 @@ ignore = [ "D402", # First word of the first line should be properly capitalized "D403", + # Use of `assert` detected; needed for mypy + "S101", + # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180 + "S321", + # Argument default set to insecure SSL protocol + "S503", ] [tool.ruff.lint.per-file-ignores] @@ -307,6 +323,9 @@ ignore = [ "scrapy/selector/__init__.py" = ["F401"] "scrapy/spiders/__init__.py" = ["E402", "F401"] +# Skip bandit in tests +"tests/**" = ["S"] + # Issues pending a review: "docs/conf.py" = ["E402"] "scrapy/utils/url.py" = ["F403", "F405"] diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index b96c63eb7..714bc38da 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,6 +1,6 @@ from __future__ import annotations -import subprocess # nosec +import subprocess import sys import time from typing import TYPE_CHECKING, Any @@ -40,9 +40,9 @@ class _BenchServer: from scrapy.utils.test import get_testenv pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"] - self.proc = subprocess.Popen( + self.proc = subprocess.Popen( # noqa: S603 pargs, stdout=subprocess.PIPE, env=get_testenv() - ) # nosec + ) assert self.proc.stdout self.proc.stdout.readline() diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 438375e02..0e046cece 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -41,4 +41,4 @@ class Command(ScrapyCommand): sfile = sys.modules[spidercls.__module__].__file__ assert sfile sfile = sfile.replace(".pyc", ".py") - self.exitcode = os.system(f'{editor} "{sfile}"') # nosec + self.exitcode = os.system(f'{editor} "{sfile}"') # noqa: S605 diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2e70b2865..38f917c7e 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -118,7 +118,7 @@ class Command(ScrapyCommand): if template_file: self._genspider(module, name, url, opts.template, template_file) if opts.edit: - self.exitcode = os.system(f'scrapy edit "{name}"') # nosec + self.exitcode = os.system(f'scrapy edit "{name}"') # noqa: S605 def _generate_template_variables( self, diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 434b316e9..78dc16df6 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -52,7 +52,7 @@ class Slot: def download_delay(self) -> float: if self.randomize_delay: - return random.uniform(0.5 * self.delay, 1.5 * self.delay) # nosec + return random.uniform(0.5 * self.delay, 1.5 * self.delay) # noqa: S311 return self.delay def close(self) -> None: diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 9fab172a8..723fe5e93 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -115,7 +115,7 @@ def get_retry_request( return new_request stats.inc_value(f"{stats_base_key}/max_reached") logger.error( - "Gave up retrying %(request)s (failed %(retry_times)d times): " "%(reason)s", + "Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s", {"request": request, "retry_times": retry_times, "reason": reason}, extra={"spider": spider}, ) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 9380b7e78..b6997ef67 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -6,13 +6,13 @@ from __future__ import annotations import csv import marshal -import pickle # nosec +import pickle import pprint from collections.abc import Callable, Iterable, Mapping from io import BytesIO, TextIOWrapper from typing import TYPE_CHECKING, Any -from xml.sax.saxutils import XMLGenerator # nosec -from xml.sax.xmlreader import AttributesImpl # nosec +from xml.sax.saxutils import XMLGenerator +from xml.sax.xmlreader import AttributesImpl from itemadapter import ItemAdapter, is_item diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index f6415ad8e..edea7cc39 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -681,7 +681,7 @@ class FeedExporter: return True except NotConfigured as e: logger.error( - "Disabled feed storage scheme: %(scheme)s. " "Reason: %(reason)s", + "Disabled feed storage scheme: %(scheme)s. Reason: %(reason)s", {"scheme": scheme, "reason": str(e)}, ) else: diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 0edcce888..965d6434b 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -2,7 +2,7 @@ from __future__ import annotations import gzip import logging -import pickle # nosec +import pickle from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path @@ -309,7 +309,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return None # expired - return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec + return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # noqa: S301 class FilesystemCacheStorage: @@ -392,7 +392,7 @@ class FilesystemCacheStorage: if 0 < self.expiration_secs < time() - mtime: return None # expired with self._open(metapath, "rb") as f: - return cast(dict[str, Any], pickle.load(f)) # nosec + return cast(dict[str, Any], pickle.load(f)) # noqa: S301 def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]: diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 642919be9..7b8756572 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,6 +1,6 @@ from __future__ import annotations -import pickle # nosec +import pickle from pathlib import Path from typing import TYPE_CHECKING @@ -41,7 +41,7 @@ class SpiderState: def spider_opened(self, spider: Spider) -> None: if self.jobdir and Path(self.statefn).exists(): with Path(self.statefn).open("rb") as f: - spider.state = pickle.load(f) # type: ignore[attr-defined] # nosec + spider.state = pickle.load(f) # type: ignore[attr-defined] # noqa: S301 else: spider.state = {} # type: ignore[attr-defined] diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index b3c3d7c7a..de3b24de0 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -11,11 +11,13 @@ from collections.abc import Iterable from typing import TYPE_CHECKING, Any, Optional, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit -from lxml.html import FormElement # nosec -from lxml.html import InputElement # nosec -from lxml.html import MultipleSelectOptions # nosec -from lxml.html import SelectElement # nosec -from lxml.html import TextareaElement # nosec +from lxml.html import ( + FormElement, + InputElement, + MultipleSelectOptions, + SelectElement, + TextareaElement, +) from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index d50388548..387805f57 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -107,7 +107,7 @@ class Response(object_ref): self._url: str = url else: raise TypeError( - f"{type(self).__name__} url must be str, " f"got {type(url).__name__}" + f"{type(self).__name__} url must be str, got {type(url).__name__}" ) @property diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index c713f6188..f954b5e9e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -308,7 +308,7 @@ def _url_from_selector(sel: parsel.Selector) -> str: raise _InvalidSelector(f"Unsupported selector: {sel}") if sel.root.tag not in ("a", "link"): raise _InvalidSelector( - "Only and elements are supported; " f"got <{sel.root.tag}>" + f"Only and elements are supported; got <{sel.root.tag}>" ) href = sel.root.get("href") if href is None: diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 192f937ce..bd96ccf19 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -12,7 +12,7 @@ from functools import partial from typing import TYPE_CHECKING, Any, Union, cast from urllib.parse import urljoin, urlparse -from lxml import etree # nosec +from lxml import etree from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string @@ -26,7 +26,7 @@ from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain if TYPE_CHECKING: - from lxml.html import HtmlElement # nosec + from lxml.html import HtmlElement from scrapy import Selector from scrapy.http import TextResponse diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index bebf6039b..16bd45c00 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -66,7 +66,7 @@ def _md5sum(file: IO[bytes]) -> str: >>> _md5sum(BytesIO(b'file content to hash')) '784406af91dd5a54fbb9c84c2236595a' """ - m = hashlib.md5() # nosec + m = hashlib.md5() # noqa: S324 while True: d = file.read(8096) if not d: @@ -399,7 +399,7 @@ class FTPFilesStore: ftp.set_pasv(False) file_path = f"{self.basedir}/{path}" last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) - m = hashlib.md5() # nosec + m = hashlib.md5() # noqa: S324 ftp.retrbinary(f"RETR {file_path}", m.update) return {"last_modified": last_modified, "checksum": m.hexdigest()} # The file doesn't exist @@ -734,7 +734,7 @@ class FilesPipeline(MediaPipeline): *, item: Any = None, ) -> str: - media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec + media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324 media_ext = Path(request.url).suffix # Handles empty and wild extensions by trying to guess the # mime type then extension or default to empty string otherwise diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e86e7c493..29dc13f0a 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -258,7 +258,7 @@ class ImagesPipeline(FilesPipeline): *, item: Any = None, ) -> str: - image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec + image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324 return f"full/{image_guid}.jpg" def thumb_path( @@ -270,5 +270,5 @@ class ImagesPipeline(FilesPipeline): *, item: Any = None, ) -> str: - thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec + thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324 return f"thumbs/{thumb_id}/{thumb_guid}.jpg" diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 4dea5afea..5b2f81335 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -33,7 +33,7 @@ def _path_safe(text: str) -> str: pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part - unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # nosec + unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # noqa: S324 return "-".join([pathable_slot, unique_slot]) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7ba0128a5..89ab21fbe 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -178,7 +178,7 @@ FILES_STORE_S3_ACL = "private" FILES_STORE_GCS_ACL = "" FTP_USER = "anonymous" -FTP_PASSWORD = "guest" # nosec +FTP_PASSWORD = "guest" # noqa: S105 FTP_PASSIVE_MODE = True GCS_PROJECT_ID = None diff --git a/scrapy/shell.py b/scrapy/shell.py index 31349c4ff..5d0ab1e4d 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -70,7 +70,7 @@ class Shell: else: self.populate_vars() if self.code: - print(eval(self.code, globals(), self.vars)) # nosec + print(eval(self.code, globals(), self.vars)) # noqa: S307 else: """ Detect interactive shell setting in scrapy.cfg diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 7732187fd..80bb37e93 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -5,7 +5,7 @@ Scheduler queues from __future__ import annotations import marshal -import pickle # nosec +import pickle from pathlib import Path from typing import TYPE_CHECKING, Any diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 550516141..923ec005e 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -15,7 +15,7 @@ class Root(Resource): def render(self, request: Request) -> bytes: total = _getarg(request, b"total", 100, int) show = _getarg(request, b"show", 10, int) - nlist = [random.randint(1, total) for _ in range(show)] # nosec + nlist = [random.randint(1, total) for _ in range(show)] # noqa: S311 request.write(b"") assert request.args is not None args = request.args.copy() diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 1430ed8d6..1948009e8 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -32,7 +32,7 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]: checks: list[tuple[str, Any]] = [] for test in tests: try: - checks += [(test, eval(test))] # nosec + checks += [(test, eval(test))] # noqa: S307 except Exception as e: checks += [(test, f"{type(e).__name__} (exception)")] diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index ba58d939c..e8ed7b60a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -7,7 +7,7 @@ from io import StringIO from typing import TYPE_CHECKING, Any, Literal, cast, overload from warnings import warn -from lxml import etree # nosec +from lxml import etree from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse @@ -41,10 +41,10 @@ def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]: nodename_patt = re.escape(nodename) - DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) - HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S) - END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S) - NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) + DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.DOTALL) + HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.DOTALL) + END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.DOTALL) + NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.DOTALL) text = _body_or_str(obj) document_header_match = re.search(DOCUMENT_HEADER_RE, text) @@ -58,7 +58,9 @@ def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]: for tagname in reversed(re.findall(END_TAG_RE, header_end)): assert header_end_idx tag = re.search( - rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S + rf"<\s*{tagname}.*?xmlns[:=][^>]*>", + text[: header_end_idx[1]], + re.DOTALL, ) if tag: for x in re.findall(NAMESPACE_RE, tag.group()): diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index eefadd07d..5ce4863f6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -116,7 +116,7 @@ def md5sum(file: IO[bytes]) -> str: ScrapyDeprecationWarning, stacklevel=2, ) - m = hashlib.md5() # nosec + m = hashlib.md5() # noqa: S324 while True: d = file.read(8096) if not d: diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index b9babb08f..511511301 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -136,7 +136,7 @@ def to_bytes( return text if not isinstance(text, str): raise TypeError( - "to_bytes must receive a str or bytes " f"object, got {type(text).__name__}" + f"to_bytes must receive a str or bytes object, got {type(text).__name__}" ) if encoding is None: encoding = "utf-8" diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 20e3151da..ad811e804 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -94,7 +94,9 @@ def fingerprint( "headers": headers, } fingerprint_json = json.dumps(fingerprint_data, sort_keys=True) - cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() # nosec + cache[cache_key] = hashlib.sha1( # noqa: S324 + fingerprint_json.encode() + ).digest() return cache[cache_key] diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 7c8ca51f2..a7ad4544d 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -104,7 +104,7 @@ def open_in_browser( elif isinstance(response, TextResponse): ext = ".txt" else: - raise TypeError("Unsupported response type: " f"{response.__class__.__name__}") + raise TypeError(f"Unsupported response type: {response.__class__.__name__}") fd, fname = tempfile.mkstemp(ext) os.write(fd, body) os.close(fd) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index c572580ae..b60fe929e 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -10,7 +10,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any from urllib.parse import urljoin -import lxml.etree # nosec +import lxml.etree if TYPE_CHECKING: from collections.abc import Iterable, Iterator @@ -24,7 +24,7 @@ class Sitemap: xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) - self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # nosec + self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # noqa: S320 rt = self._root.tag self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index a5cc22c1c..2539f30c7 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -89,7 +89,7 @@ def escape_ajax(url: str) -> str: def add_http_if_no_scheme(url: str) -> str: """Add http as the default scheme if it is missing from the url.""" - match = re.match(r"^\w+://", url, flags=re.I) + match = re.match(r"^\w+://", url, flags=re.IGNORECASE) if not match: parts = urlparse(url) scheme = "http:" if parts.netloc else "http://" diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 4e9e29286..996a5cdb3 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -3,7 +3,7 @@ import sys import cryptography import cssselect -import lxml.etree # nosec +import lxml.etree import parsel import twisted import w3lib diff --git a/tests/test_commands.py b/tests/test_commands.py index e7df7b6e8..32b69de8a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -238,7 +238,7 @@ class StartprojectTemplatesTest(ProjectTest): args = ["--set", f"TEMPLATES_DIR={self.tmpl}"] p, out, err = self.proc("startproject", self.project_name, *args) self.assertIn( - f"New Scrapy project '{self.project_name}', " "using template directory", + f"New Scrapy project '{self.project_name}', using template directory", out, ) self.assertIn(self.tmpl_proj, out) diff --git a/tests/test_engine.py b/tests/test_engine.py index 2ebc0b5e4..8d645eada 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -67,8 +67,8 @@ class TestSpider(Spider): allowed_domains = ["scrapytest.org", "localhost"] itemurl_re = re.compile(r"item\d+.html") - name_re = re.compile(r"

(.*?)

", re.M) - price_re = re.compile(r">Price: \$(.*?)<", re.M) + name_re = re.compile(r"

(.*?)

", re.MULTILINE) + price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE) item_cls: type = TestItem diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 96a2c42b7..2be5e09bc 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -631,7 +631,7 @@ class TestGCSFilesStore(unittest.TestCase): """ assert_gcs_environ() try: - import google.cloud.storage # noqa + import google.cloud.storage # noqa: F401 except ModuleNotFoundError: raise unittest.SkipTest("google-cloud-storage is not installed") with mock.patch("google.cloud.storage") as _: diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 541979dcc..e127cc2e3 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -66,14 +66,14 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second")) def test_length_based_precedence(self): - robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page" + robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) self.assertTrue(rp.allowed("https://www.site.local/page", "*")) def test_order_based_precedence(self): - robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page" + robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 35d1508c6..ef07d625f 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -21,9 +21,9 @@ class TrackrefTestCase(unittest.TestCase): trackref.live_refs.clear() def test_format_live_refs(self): - o1 = Foo() # NOQA - o2 = Bar() # NOQA - o3 = Foo() # NOQA + o1 = Foo() # noqa: F841 + o2 = Bar() # noqa: F841 + o3 = Foo() # noqa: F841 self.assertEqual( trackref.format_live_refs(), """\ @@ -50,7 +50,7 @@ Bar 1 oldest: 0s ago @mock.patch("sys.stdout", new_callable=StringIO) def test_print_live_refs_with_objects(self, stdout): - o1 = Foo() # NOQA + o1 = Foo() # noqa: F841 trackref.print_live_refs() self.assertEqual( stdout.getvalue(), @@ -61,11 +61,11 @@ Foo 1 oldest: 0s ago\n\n""", ) def test_get_oldest(self): - o1 = Foo() # NOQA + o1 = Foo() # noqa: F841 o1_time = time() - o2 = Bar() # NOQA + o2 = Bar() # noqa: F841 o3_time = time() if o3_time <= o1_time: @@ -74,15 +74,15 @@ Foo 1 oldest: 0s ago\n\n""", if o3_time <= o1_time: raise SkipTest("time.time is not precise enough") - o3 = Foo() # NOQA + o3 = Foo() # noqa: F841 self.assertIs(trackref.get_oldest("Foo"), o1) self.assertIs(trackref.get_oldest("Bar"), o2) self.assertIsNone(trackref.get_oldest("XXX")) def test_iter_all(self): - o1 = Foo() # NOQA - o2 = Bar() # NOQA - o3 = Foo() # NOQA + o1 = Foo() # noqa: F841 + o2 = Bar() # noqa: F841 + o3 = Foo() # noqa: F841 self.assertEqual( set(trackref.iter_all("Foo")), {o1, o3}, diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 1cad68b9c..0a594aa7c 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -161,7 +161,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): # test minimal sent headers factory = client.ScrapyHTTPClientFactory(Request("http://foo/bar")) - self._test(factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"\r\n") + self._test(factory, b"GET /bar HTTP/1.0\r\nHost: foo\r\n\r\n") # test a simple POST with body and content-type factory = client.ScrapyHTTPClientFactory( @@ -191,7 +191,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): self._test( factory, - b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Content-Length: 0\r\n" b"\r\n", + b"POST /bar HTTP/1.0\r\nHost: foo\r\nContent-Length: 0\r\n\r\n", ) # test with single and multivalued headers