From 2bfd9a2257c79ae56955e95b46f2bc7b23e1eabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 29 Feb 2024 11:11:42 +0100 Subject: [PATCH] bandit: allow-list false positives --- .bandit.yml | 11 +---------- scrapy/commands/bench.py | 6 ++++-- scrapy/commands/edit.py | 2 +- scrapy/commands/genspider.py | 2 +- scrapy/core/downloader/__init__.py | 2 +- scrapy/exporters.py | 4 ++-- scrapy/extensions/httpcache.py | 6 +++--- scrapy/extensions/spiderstate.py | 4 ++-- scrapy/settings/default_settings.py | 2 +- scrapy/shell.py | 2 +- scrapy/squeues.py | 2 +- scrapy/utils/benchserver.py | 2 +- scrapy/utils/engine.py | 2 +- 13 files changed, 20 insertions(+), 27 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 6e8331c0f..4fcd75c57 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,19 +1,10 @@ skips: -- B101 # assert_used -- B105 # hardcoded_password_string -- B301 # pickle -- B307 # eval -- B311 # random +- B101 # assert_used, needed for mypy - B320 # xml_bad_etree - B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 - B324 # hashlib "Use of weak SHA1 hash for security" - B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B403 # import_pickle -- B404 # import_subprocess -- B406 # import_xml_sax - B410 # import_lxml - B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 - B503 # ssl_with_bad_defaults -- B603 # subprocess_without_shell_equals_true -- B605 # start_process_with_a_shell exclude_dirs: ['tests'] diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index e1ccdc451..aaf5a439f 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,4 +1,4 @@ -import subprocess +import subprocess # nosec import sys import time from urllib.parse import urlencode @@ -29,7 +29,9 @@ class _BenchServer: from scrapy.utils.test import get_testenv pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"] - self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv()) + self.proc = subprocess.Popen( + pargs, stdout=subprocess.PIPE, env=get_testenv() + ) # nosec self.proc.stdout.readline() def __exit__(self, exc_type, exc_value, traceback): diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 03a8ed5c7..e85d2c9ec 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -37,4 +37,4 @@ class Command(ScrapyCommand): sfile = sys.modules[spidercls.__module__].__file__ sfile = sfile.replace(".pyc", ".py") - self.exitcode = os.system(f'{editor} "{sfile}"') + self.exitcode = os.system(f'{editor} "{sfile}"') # nosec diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 68cbe8ff6..567ebcdc0 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -113,7 +113,7 @@ class Command(ScrapyCommand): if template_file: self._genspider(module, name, url, opts.template, template_file) if opts.edit: - self.exitcode = os.system(f'scrapy edit "{name}"') + self.exitcode = os.system(f'scrapy edit "{name}"') # nosec def _genspider(self, module, name, url, template_name, template_file): """Generate the spider module, based on the given template""" diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index c84525160..666282856 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -40,7 +40,7 @@ class Slot: def download_delay(self) -> float: if self.randomize_delay: - return random.uniform(0.5 * self.delay, 1.5 * self.delay) + return random.uniform(0.5 * self.delay, 1.5 * self.delay) # nosec return self.delay def close(self) -> None: diff --git a/scrapy/exporters.py b/scrapy/exporters.py index f85f1dad8..79fd4e56f 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -5,10 +5,10 @@ Item Exporters are used to export/serialize items into different formats. import csv import io import marshal -import pickle +import pickle # nosec import pprint from collections.abc import Mapping -from xml.sax.saxutils import XMLGenerator +from xml.sax.saxutils import XMLGenerator # nosec from itemadapter import ItemAdapter, is_item diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 7e4f047a8..335728502 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,6 +1,6 @@ import gzip import logging -import pickle +import pickle # nosec from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path @@ -274,7 +274,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return # expired - return pickle.loads(db[f"{key}_data"]) + return pickle.loads(db[f"{key}_data"]) # nosec class FilesystemCacheStorage: @@ -352,7 +352,7 @@ class FilesystemCacheStorage: if 0 < self.expiration_secs < time() - mtime: return # expired with self._open(metapath, "rb") as f: - return pickle.load(f) + return pickle.load(f) # nosec def parse_cachecontrol(header): diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 929a3be70..43359401b 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,4 +1,4 @@ -import pickle +import pickle # nosec from pathlib import Path from scrapy import signals @@ -31,7 +31,7 @@ class SpiderState: def spider_opened(self, spider): if self.jobdir and Path(self.statefn).exists(): with Path(self.statefn).open("rb") as f: - spider.state = pickle.load(f) + spider.state = pickle.load(f) # nosec else: spider.state = {} diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 49ab1b5ef..2b3d95a0e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -177,7 +177,7 @@ FILES_STORE_S3_ACL = "private" FILES_STORE_GCS_ACL = "" FTP_USER = "anonymous" -FTP_PASSWORD = "guest" +FTP_PASSWORD = "guest" # nosec FTP_PASSIVE_MODE = True GCS_PROJECT_ID = None diff --git a/scrapy/shell.py b/scrapy/shell.py index 05909977a..63ea33892 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -50,7 +50,7 @@ class Shell: else: self.populate_vars() if self.code: - print(eval(self.code, globals(), self.vars)) + print(eval(self.code, globals(), self.vars)) # nosec else: """ Detect interactive shell setting in scrapy.cfg diff --git a/scrapy/squeues.py b/scrapy/squeues.py index f665ad88c..e20f60f06 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -3,7 +3,7 @@ Scheduler queues """ import marshal -import pickle +import pickle # nosec from os import PathLike from pathlib import Path from typing import Union diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 38884a9f0..f6f704d4b 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -14,7 +14,7 @@ class Root(Resource): def render(self, request): total = _getarg(request, b"total", 100, int) show = _getarg(request, b"show", 10, int) - nlist = [random.randint(1, total) for _ in range(show)] + nlist = [random.randint(1, total) for _ in range(show)] # nosec request.write(b"") args = request.args.copy() for nl in nlist: diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index a5f2a8c6e..0b2722663 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -30,7 +30,7 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: checks: List[Tuple[str, Any]] = [] for test in tests: try: - checks += [(test, eval(test))] + checks += [(test, eval(test))] # nosec except Exception as e: checks += [(test, f"{type(e).__name__} (exception)")]