diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 49db3f610..b273e269b 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,12 +3,6 @@ repos:
rev: v0.8.1
hooks:
- id: ruff
-- repo: https://github.com/PyCQA/bandit
- rev: 1.7.9
- hooks:
- - id: bandit
- args: ["-c", "pyproject.toml"]
- additional_dependencies: ["bandit[toml]"]
- repo: https://github.com/psf/black.git
rev: 24.4.2
hooks:
@@ -23,8 +17,3 @@ repos:
- id: blacken-docs
additional_dependencies:
- black==24.4.2
-- repo: https://github.com/asottile/pyupgrade
- rev: v3.18.0
- hooks:
- - id: pyupgrade
- args: [--py39-plus]
diff --git a/pyproject.toml b/pyproject.toml
index 1378bab50..977792178 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -92,16 +92,6 @@ follow_imports = "skip"
module = "scrapy.settings.default_settings"
ignore_errors = true
-[tool.bandit]
-skips = [
- "B101", # assert_used, needed for mypy
- "B321", # ftplib, https://github.com/scrapy/scrapy/issues/4180
- "B402", # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
- "B411", # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
- "B503", # ssl_with_bad_defaults
-]
-exclude_dirs = ["tests"]
-
[tool.bumpversion]
current_version = "2.12.0"
commit = true
@@ -242,10 +232,30 @@ extend-select = [
"C4",
# pydocstyle
"D",
+ # flake8-future-annotations
+ "FA",
+ # refurb
+ "FURB",
+ # flake8-implicit-str-concat
+ "ISC",
+ # flake8-logging
+ "LOG",
+ # pygrep-hooks
+ "PGH",
+ # flake8-quotes
+ "Q",
+ # flake8-bandit
+ "S",
+ # flake8-slots
+ "SLOT",
# flake8-debugger
"T10",
# flake8-type-checking
"TC",
+ # pyupgrade
+ "UP",
+ # flake8-2020
+ "YTT",
]
ignore = [
# Assigning to `os.environ` doesn't clear the environment.
@@ -296,6 +306,12 @@ ignore = [
"D402",
# First word of the first line should be properly capitalized
"D403",
+ # Use of `assert` detected; needed for mypy
+ "S101",
+ # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180
+ "S321",
+ # Argument default set to insecure SSL protocol
+ "S503",
]
[tool.ruff.lint.per-file-ignores]
@@ -307,6 +323,9 @@ ignore = [
"scrapy/selector/__init__.py" = ["F401"]
"scrapy/spiders/__init__.py" = ["E402", "F401"]
+# Skip bandit in tests
+"tests/**" = ["S"]
+
# Issues pending a review:
"docs/conf.py" = ["E402"]
"scrapy/utils/url.py" = ["F403", "F405"]
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index b96c63eb7..714bc38da 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-import subprocess # nosec
+import subprocess
import sys
import time
from typing import TYPE_CHECKING, Any
@@ -40,9 +40,9 @@ class _BenchServer:
from scrapy.utils.test import get_testenv
pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
- self.proc = subprocess.Popen(
+ self.proc = subprocess.Popen( # noqa: S603
pargs, stdout=subprocess.PIPE, env=get_testenv()
- ) # nosec
+ )
assert self.proc.stdout
self.proc.stdout.readline()
diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index 438375e02..0e046cece 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -41,4 +41,4 @@ class Command(ScrapyCommand):
sfile = sys.modules[spidercls.__module__].__file__
assert sfile
sfile = sfile.replace(".pyc", ".py")
- self.exitcode = os.system(f'{editor} "{sfile}"') # nosec
+ self.exitcode = os.system(f'{editor} "{sfile}"') # noqa: S605
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 2e70b2865..38f917c7e 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -118,7 +118,7 @@ class Command(ScrapyCommand):
if template_file:
self._genspider(module, name, url, opts.template, template_file)
if opts.edit:
- self.exitcode = os.system(f'scrapy edit "{name}"') # nosec
+ self.exitcode = os.system(f'scrapy edit "{name}"') # noqa: S605
def _generate_template_variables(
self,
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 434b316e9..78dc16df6 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -52,7 +52,7 @@ class Slot:
def download_delay(self) -> float:
if self.randomize_delay:
- return random.uniform(0.5 * self.delay, 1.5 * self.delay) # nosec
+ return random.uniform(0.5 * self.delay, 1.5 * self.delay) # noqa: S311
return self.delay
def close(self) -> None:
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 9fab172a8..723fe5e93 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -115,7 +115,7 @@ def get_retry_request(
return new_request
stats.inc_value(f"{stats_base_key}/max_reached")
logger.error(
- "Gave up retrying %(request)s (failed %(retry_times)d times): " "%(reason)s",
+ "Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s",
{"request": request, "retry_times": retry_times, "reason": reason},
extra={"spider": spider},
)
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 9380b7e78..b6997ef67 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -6,13 +6,13 @@ from __future__ import annotations
import csv
import marshal
-import pickle # nosec
+import pickle
import pprint
from collections.abc import Callable, Iterable, Mapping
from io import BytesIO, TextIOWrapper
from typing import TYPE_CHECKING, Any
-from xml.sax.saxutils import XMLGenerator # nosec
-from xml.sax.xmlreader import AttributesImpl # nosec
+from xml.sax.saxutils import XMLGenerator
+from xml.sax.xmlreader import AttributesImpl
from itemadapter import ItemAdapter, is_item
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index f6415ad8e..edea7cc39 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -681,7 +681,7 @@ class FeedExporter:
return True
except NotConfigured as e:
logger.error(
- "Disabled feed storage scheme: %(scheme)s. " "Reason: %(reason)s",
+ "Disabled feed storage scheme: %(scheme)s. Reason: %(reason)s",
{"scheme": scheme, "reason": str(e)},
)
else:
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 0edcce888..965d6434b 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import gzip
import logging
-import pickle # nosec
+import pickle
from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from pathlib import Path
@@ -309,7 +309,7 @@ class DbmCacheStorage:
if 0 < self.expiration_secs < time() - float(ts):
return None # expired
- return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec
+ return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # noqa: S301
class FilesystemCacheStorage:
@@ -392,7 +392,7 @@ class FilesystemCacheStorage:
if 0 < self.expiration_secs < time() - mtime:
return None # expired
with self._open(metapath, "rb") as f:
- return cast(dict[str, Any], pickle.load(f)) # nosec
+ return cast(dict[str, Any], pickle.load(f)) # noqa: S301
def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]:
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 642919be9..7b8756572 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-import pickle # nosec
+import pickle
from pathlib import Path
from typing import TYPE_CHECKING
@@ -41,7 +41,7 @@ class SpiderState:
def spider_opened(self, spider: Spider) -> None:
if self.jobdir and Path(self.statefn).exists():
with Path(self.statefn).open("rb") as f:
- spider.state = pickle.load(f) # type: ignore[attr-defined] # nosec
+ spider.state = pickle.load(f) # type: ignore[attr-defined] # noqa: S301
else:
spider.state = {} # type: ignore[attr-defined]
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index b3c3d7c7a..de3b24de0 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -11,11 +11,13 @@ from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, Optional, Union, cast
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
-from lxml.html import FormElement # nosec
-from lxml.html import InputElement # nosec
-from lxml.html import MultipleSelectOptions # nosec
-from lxml.html import SelectElement # nosec
-from lxml.html import TextareaElement # nosec
+from lxml.html import (
+ FormElement,
+ InputElement,
+ MultipleSelectOptions,
+ SelectElement,
+ TextareaElement,
+)
from w3lib.html import strip_html5_whitespace
from scrapy.http.request import Request
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index d50388548..387805f57 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -107,7 +107,7 @@ class Response(object_ref):
self._url: str = url
else:
raise TypeError(
- f"{type(self).__name__} url must be str, " f"got {type(url).__name__}"
+ f"{type(self).__name__} url must be str, got {type(url).__name__}"
)
@property
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index c713f6188..f954b5e9e 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -308,7 +308,7 @@ def _url_from_selector(sel: parsel.Selector) -> str:
raise _InvalidSelector(f"Unsupported selector: {sel}")
if sel.root.tag not in ("a", "link"):
raise _InvalidSelector(
- "Only and elements are supported; " f"got <{sel.root.tag}>"
+ f"Only and elements are supported; got <{sel.root.tag}>"
)
href = sel.root.get("href")
if href is None:
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 192f937ce..bd96ccf19 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -12,7 +12,7 @@ from functools import partial
from typing import TYPE_CHECKING, Any, Union, cast
from urllib.parse import urljoin, urlparse
-from lxml import etree # nosec
+from lxml import etree
from parsel.csstranslator import HTMLTranslator
from w3lib.html import strip_html5_whitespace
from w3lib.url import canonicalize_url, safe_url_string
@@ -26,7 +26,7 @@ from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain
if TYPE_CHECKING:
- from lxml.html import HtmlElement # nosec
+ from lxml.html import HtmlElement
from scrapy import Selector
from scrapy.http import TextResponse
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index bebf6039b..16bd45c00 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -66,7 +66,7 @@ def _md5sum(file: IO[bytes]) -> str:
>>> _md5sum(BytesIO(b'file content to hash'))
'784406af91dd5a54fbb9c84c2236595a'
"""
- m = hashlib.md5() # nosec
+ m = hashlib.md5() # noqa: S324
while True:
d = file.read(8096)
if not d:
@@ -399,7 +399,7 @@ class FTPFilesStore:
ftp.set_pasv(False)
file_path = f"{self.basedir}/{path}"
last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())
- m = hashlib.md5() # nosec
+ m = hashlib.md5() # noqa: S324
ftp.retrbinary(f"RETR {file_path}", m.update)
return {"last_modified": last_modified, "checksum": m.hexdigest()}
# The file doesn't exist
@@ -734,7 +734,7 @@ class FilesPipeline(MediaPipeline):
*,
item: Any = None,
) -> str:
- media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec
+ media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324
media_ext = Path(request.url).suffix
# Handles empty and wild extensions by trying to guess the
# mime type then extension or default to empty string otherwise
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index e86e7c493..29dc13f0a 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -258,7 +258,7 @@ class ImagesPipeline(FilesPipeline):
*,
item: Any = None,
) -> str:
- image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec
+ image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324
return f"full/{image_guid}.jpg"
def thumb_path(
@@ -270,5 +270,5 @@ class ImagesPipeline(FilesPipeline):
*,
item: Any = None,
) -> str:
- thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec
+ thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324
return f"thumbs/{thumb_id}/{thumb_guid}.jpg"
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 4dea5afea..5b2f81335 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -33,7 +33,7 @@ def _path_safe(text: str) -> str:
pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text])
# as we replace some letters we can get collision for different slots
# add we add unique part
- unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # nosec
+ unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # noqa: S324
return "-".join([pathable_slot, unique_slot])
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 7ba0128a5..89ab21fbe 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -178,7 +178,7 @@ FILES_STORE_S3_ACL = "private"
FILES_STORE_GCS_ACL = ""
FTP_USER = "anonymous"
-FTP_PASSWORD = "guest" # nosec
+FTP_PASSWORD = "guest" # noqa: S105
FTP_PASSIVE_MODE = True
GCS_PROJECT_ID = None
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 31349c4ff..5d0ab1e4d 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -70,7 +70,7 @@ class Shell:
else:
self.populate_vars()
if self.code:
- print(eval(self.code, globals(), self.vars)) # nosec
+ print(eval(self.code, globals(), self.vars)) # noqa: S307
else:
"""
Detect interactive shell setting in scrapy.cfg
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 7732187fd..80bb37e93 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -5,7 +5,7 @@ Scheduler queues
from __future__ import annotations
import marshal
-import pickle # nosec
+import pickle
from pathlib import Path
from typing import TYPE_CHECKING, Any
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 550516141..923ec005e 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -15,7 +15,7 @@ class Root(Resource):
def render(self, request: Request) -> bytes:
total = _getarg(request, b"total", 100, int)
show = _getarg(request, b"show", 10, int)
- nlist = [random.randint(1, total) for _ in range(show)] # nosec
+ nlist = [random.randint(1, total) for _ in range(show)] # noqa: S311
request.write(b"")
assert request.args is not None
args = request.args.copy()
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index 1430ed8d6..1948009e8 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -32,7 +32,7 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]:
checks: list[tuple[str, Any]] = []
for test in tests:
try:
- checks += [(test, eval(test))] # nosec
+ checks += [(test, eval(test))] # noqa: S307
except Exception as e:
checks += [(test, f"{type(e).__name__} (exception)")]
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index ba58d939c..e8ed7b60a 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -7,7 +7,7 @@ from io import StringIO
from typing import TYPE_CHECKING, Any, Literal, cast, overload
from warnings import warn
-from lxml import etree # nosec
+from lxml import etree
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Response, TextResponse
@@ -41,10 +41,10 @@ def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]:
nodename_patt = re.escape(nodename)
- DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S)
- HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S)
- END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S)
- NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S)
+ DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.DOTALL)
+ HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.DOTALL)
+ END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.DOTALL)
+ NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.DOTALL)
text = _body_or_str(obj)
document_header_match = re.search(DOCUMENT_HEADER_RE, text)
@@ -58,7 +58,9 @@ def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]:
for tagname in reversed(re.findall(END_TAG_RE, header_end)):
assert header_end_idx
tag = re.search(
- rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S
+ rf"<\s*{tagname}.*?xmlns[:=][^>]*>",
+ text[: header_end_idx[1]],
+ re.DOTALL,
)
if tag:
for x in re.findall(NAMESPACE_RE, tag.group()):
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index eefadd07d..5ce4863f6 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -116,7 +116,7 @@ def md5sum(file: IO[bytes]) -> str:
ScrapyDeprecationWarning,
stacklevel=2,
)
- m = hashlib.md5() # nosec
+ m = hashlib.md5() # noqa: S324
while True:
d = file.read(8096)
if not d:
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index b9babb08f..511511301 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -136,7 +136,7 @@ def to_bytes(
return text
if not isinstance(text, str):
raise TypeError(
- "to_bytes must receive a str or bytes " f"object, got {type(text).__name__}"
+ f"to_bytes must receive a str or bytes object, got {type(text).__name__}"
)
if encoding is None:
encoding = "utf-8"
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 20e3151da..ad811e804 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -94,7 +94,9 @@ def fingerprint(
"headers": headers,
}
fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
- cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() # nosec
+ cache[cache_key] = hashlib.sha1( # noqa: S324
+ fingerprint_json.encode()
+ ).digest()
return cache[cache_key]
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 7c8ca51f2..a7ad4544d 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -104,7 +104,7 @@ def open_in_browser(
elif isinstance(response, TextResponse):
ext = ".txt"
else:
- raise TypeError("Unsupported response type: " f"{response.__class__.__name__}")
+ raise TypeError(f"Unsupported response type: {response.__class__.__name__}")
fd, fname = tempfile.mkstemp(ext)
os.write(fd, body)
os.close(fd)
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index c572580ae..b60fe929e 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -10,7 +10,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any
from urllib.parse import urljoin
-import lxml.etree # nosec
+import lxml.etree
if TYPE_CHECKING:
from collections.abc import Iterable, Iterator
@@ -24,7 +24,7 @@ class Sitemap:
xmlp = lxml.etree.XMLParser(
recover=True, remove_comments=True, resolve_entities=False
)
- self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # nosec
+ self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # noqa: S320
rt = self._root.tag
self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index a5cc22c1c..2539f30c7 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -89,7 +89,7 @@ def escape_ajax(url: str) -> str:
def add_http_if_no_scheme(url: str) -> str:
"""Add http as the default scheme if it is missing from the url."""
- match = re.match(r"^\w+://", url, flags=re.I)
+ match = re.match(r"^\w+://", url, flags=re.IGNORECASE)
if not match:
parts = urlparse(url)
scheme = "http:" if parts.netloc else "http://"
diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py
index 4e9e29286..996a5cdb3 100644
--- a/scrapy/utils/versions.py
+++ b/scrapy/utils/versions.py
@@ -3,7 +3,7 @@ import sys
import cryptography
import cssselect
-import lxml.etree # nosec
+import lxml.etree
import parsel
import twisted
import w3lib
diff --git a/tests/test_commands.py b/tests/test_commands.py
index e7df7b6e8..32b69de8a 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -238,7 +238,7 @@ class StartprojectTemplatesTest(ProjectTest):
args = ["--set", f"TEMPLATES_DIR={self.tmpl}"]
p, out, err = self.proc("startproject", self.project_name, *args)
self.assertIn(
- f"New Scrapy project '{self.project_name}', " "using template directory",
+ f"New Scrapy project '{self.project_name}', using template directory",
out,
)
self.assertIn(self.tmpl_proj, out)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 2ebc0b5e4..8d645eada 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -67,8 +67,8 @@ class TestSpider(Spider):
allowed_domains = ["scrapytest.org", "localhost"]
itemurl_re = re.compile(r"item\d+.html")
- name_re = re.compile(r"(.*?)
", re.M)
- price_re = re.compile(r">Price: \$(.*?)<", re.M)
+ name_re = re.compile(r"(.*?)
", re.MULTILINE)
+ price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE)
item_cls: type = TestItem
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 96a2c42b7..2be5e09bc 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -631,7 +631,7 @@ class TestGCSFilesStore(unittest.TestCase):
"""
assert_gcs_environ()
try:
- import google.cloud.storage # noqa
+ import google.cloud.storage # noqa: F401
except ModuleNotFoundError:
raise unittest.SkipTest("google-cloud-storage is not installed")
with mock.patch("google.cloud.storage") as _:
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 541979dcc..e127cc2e3 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -66,14 +66,14 @@ class BaseRobotParserTest:
self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second"))
def test_length_based_precedence(self):
- robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"
+ robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
self.assertTrue(rp.allowed("https://www.site.local/page", "*"))
def test_order_based_precedence(self):
- robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"
+ robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index 35d1508c6..ef07d625f 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -21,9 +21,9 @@ class TrackrefTestCase(unittest.TestCase):
trackref.live_refs.clear()
def test_format_live_refs(self):
- o1 = Foo() # NOQA
- o2 = Bar() # NOQA
- o3 = Foo() # NOQA
+ o1 = Foo() # noqa: F841
+ o2 = Bar() # noqa: F841
+ o3 = Foo() # noqa: F841
self.assertEqual(
trackref.format_live_refs(),
"""\
@@ -50,7 +50,7 @@ Bar 1 oldest: 0s ago
@mock.patch("sys.stdout", new_callable=StringIO)
def test_print_live_refs_with_objects(self, stdout):
- o1 = Foo() # NOQA
+ o1 = Foo() # noqa: F841
trackref.print_live_refs()
self.assertEqual(
stdout.getvalue(),
@@ -61,11 +61,11 @@ Foo 1 oldest: 0s ago\n\n""",
)
def test_get_oldest(self):
- o1 = Foo() # NOQA
+ o1 = Foo() # noqa: F841
o1_time = time()
- o2 = Bar() # NOQA
+ o2 = Bar() # noqa: F841
o3_time = time()
if o3_time <= o1_time:
@@ -74,15 +74,15 @@ Foo 1 oldest: 0s ago\n\n""",
if o3_time <= o1_time:
raise SkipTest("time.time is not precise enough")
- o3 = Foo() # NOQA
+ o3 = Foo() # noqa: F841
self.assertIs(trackref.get_oldest("Foo"), o1)
self.assertIs(trackref.get_oldest("Bar"), o2)
self.assertIsNone(trackref.get_oldest("XXX"))
def test_iter_all(self):
- o1 = Foo() # NOQA
- o2 = Bar() # NOQA
- o3 = Foo() # NOQA
+ o1 = Foo() # noqa: F841
+ o2 = Bar() # noqa: F841
+ o3 = Foo() # noqa: F841
self.assertEqual(
set(trackref.iter_all("Foo")),
{o1, o3},
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 1cad68b9c..0a594aa7c 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -161,7 +161,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
# test minimal sent headers
factory = client.ScrapyHTTPClientFactory(Request("http://foo/bar"))
- self._test(factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"\r\n")
+ self._test(factory, b"GET /bar HTTP/1.0\r\nHost: foo\r\n\r\n")
# test a simple POST with body and content-type
factory = client.ScrapyHTTPClientFactory(
@@ -191,7 +191,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
self._test(
factory,
- b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Content-Length: 0\r\n" b"\r\n",
+ b"POST /bar HTTP/1.0\r\nHost: foo\r\nContent-Length: 0\r\n\r\n",
)
# test with single and multivalued headers