mirror of https://github.com/scrapy/scrapy.git
Ruff: migrate pyupgrade and bandit, enable some other rules (#6577)
This commit is contained in:
Andrey Rakhmatullin
GitHub
parent
b423e971ae
commit
cde0845ab2
|
|
@ -3,12 +3,6 @@ repos:
|
|||
rev: v0.8.1
|
||||
hooks:
|
||||
- id: ruff
|
||||
- repo: https://github.com/PyCQA/bandit
|
||||
rev: 1.7.9
|
||||
hooks:
|
||||
- id: bandit
|
||||
args: ["-c", "pyproject.toml"]
|
||||
additional_dependencies: ["bandit[toml]"]
|
||||
- repo: https://github.com/psf/black.git
|
||||
rev: 24.4.2
|
||||
hooks:
|
||||
|
|
@ -23,8 +17,3 @@ repos:
|
|||
- id: blacken-docs
|
||||
additional_dependencies:
|
||||
- black==24.4.2
|
||||
- repo: https://github.com/asottile/pyupgrade
|
||||
rev: v3.18.0
|
||||
hooks:
|
||||
- id: pyupgrade
|
||||
args: [--py39-plus]
|
||||
|
|
|
|||
|
|
@ -92,16 +92,6 @@ follow_imports = "skip"
|
|||
module = "scrapy.settings.default_settings"
|
||||
ignore_errors = true
|
||||
|
||||
[tool.bandit]
|
||||
skips = [
|
||||
"B101", # assert_used, needed for mypy
|
||||
"B321", # ftplib, https://github.com/scrapy/scrapy/issues/4180
|
||||
"B402", # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
|
||||
"B411", # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
|
||||
"B503", # ssl_with_bad_defaults
|
||||
]
|
||||
exclude_dirs = ["tests"]
|
||||
|
||||
[tool.bumpversion]
|
||||
current_version = "2.12.0"
|
||||
commit = true
|
||||
|
|
@ -242,10 +232,30 @@ extend-select = [
|
|||
"C4",
|
||||
# pydocstyle
|
||||
"D",
|
||||
# flake8-future-annotations
|
||||
"FA",
|
||||
# refurb
|
||||
"FURB",
|
||||
# flake8-implicit-str-concat
|
||||
"ISC",
|
||||
# flake8-logging
|
||||
"LOG",
|
||||
# pygrep-hooks
|
||||
"PGH",
|
||||
# flake8-quotes
|
||||
"Q",
|
||||
# flake8-bandit
|
||||
"S",
|
||||
# flake8-slots
|
||||
"SLOT",
|
||||
# flake8-debugger
|
||||
"T10",
|
||||
# flake8-type-checking
|
||||
"TC",
|
||||
# pyupgrade
|
||||
"UP",
|
||||
# flake8-2020
|
||||
"YTT",
|
||||
]
|
||||
ignore = [
|
||||
# Assigning to `os.environ` doesn't clear the environment.
|
||||
|
|
@ -296,6 +306,12 @@ ignore = [
|
|||
"D402",
|
||||
# First word of the first line should be properly capitalized
|
||||
"D403",
|
||||
# Use of `assert` detected; needed for mypy
|
||||
"S101",
|
||||
# FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180
|
||||
"S321",
|
||||
# Argument default set to insecure SSL protocol
|
||||
"S503",
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
|
|
@ -307,6 +323,9 @@ ignore = [
|
|||
"scrapy/selector/__init__.py" = ["F401"]
|
||||
"scrapy/spiders/__init__.py" = ["E402", "F401"]
|
||||
|
||||
# Skip bandit in tests
|
||||
"tests/**" = ["S"]
|
||||
|
||||
# Issues pending a review:
|
||||
"docs/conf.py" = ["E402"]
|
||||
"scrapy/utils/url.py" = ["F403", "F405"]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import subprocess # nosec
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
|
@ -40,9 +40,9 @@ class _BenchServer:
|
|||
from scrapy.utils.test import get_testenv
|
||||
|
||||
pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
|
||||
self.proc = subprocess.Popen(
|
||||
self.proc = subprocess.Popen( # noqa: S603
|
||||
pargs, stdout=subprocess.PIPE, env=get_testenv()
|
||||
) # nosec
|
||||
)
|
||||
assert self.proc.stdout
|
||||
self.proc.stdout.readline()
|
||||
|
||||
|
|
|
|||
|
|
@ -41,4 +41,4 @@ class Command(ScrapyCommand):
|
|||
sfile = sys.modules[spidercls.__module__].__file__
|
||||
assert sfile
|
||||
sfile = sfile.replace(".pyc", ".py")
|
||||
self.exitcode = os.system(f'{editor} "{sfile}"') # nosec
|
||||
self.exitcode = os.system(f'{editor} "{sfile}"') # noqa: S605
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ class Command(ScrapyCommand):
|
|||
if template_file:
|
||||
self._genspider(module, name, url, opts.template, template_file)
|
||||
if opts.edit:
|
||||
self.exitcode = os.system(f'scrapy edit "{name}"') # nosec
|
||||
self.exitcode = os.system(f'scrapy edit "{name}"') # noqa: S605
|
||||
|
||||
def _generate_template_variables(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ class Slot:
|
|||
|
||||
def download_delay(self) -> float:
|
||||
if self.randomize_delay:
|
||||
return random.uniform(0.5 * self.delay, 1.5 * self.delay) # nosec
|
||||
return random.uniform(0.5 * self.delay, 1.5 * self.delay) # noqa: S311
|
||||
return self.delay
|
||||
|
||||
def close(self) -> None:
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@ def get_retry_request(
|
|||
return new_request
|
||||
stats.inc_value(f"{stats_base_key}/max_reached")
|
||||
logger.error(
|
||||
"Gave up retrying %(request)s (failed %(retry_times)d times): " "%(reason)s",
|
||||
"Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s",
|
||||
{"request": request, "retry_times": retry_times, "reason": reason},
|
||||
extra={"spider": spider},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,13 +6,13 @@ from __future__ import annotations
|
|||
|
||||
import csv
|
||||
import marshal
|
||||
import pickle # nosec
|
||||
import pickle
|
||||
import pprint
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from io import BytesIO, TextIOWrapper
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from xml.sax.saxutils import XMLGenerator # nosec
|
||||
from xml.sax.xmlreader import AttributesImpl # nosec
|
||||
from xml.sax.saxutils import XMLGenerator
|
||||
from xml.sax.xmlreader import AttributesImpl
|
||||
|
||||
from itemadapter import ItemAdapter, is_item
|
||||
|
||||
|
|
|
|||
|
|
@ -681,7 +681,7 @@ class FeedExporter:
|
|||
return True
|
||||
except NotConfigured as e:
|
||||
logger.error(
|
||||
"Disabled feed storage scheme: %(scheme)s. " "Reason: %(reason)s",
|
||||
"Disabled feed storage scheme: %(scheme)s. Reason: %(reason)s",
|
||||
{"scheme": scheme, "reason": str(e)},
|
||||
)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import gzip
|
||||
import logging
|
||||
import pickle # nosec
|
||||
import pickle
|
||||
from email.utils import mktime_tz, parsedate_tz
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
|
|
@ -309,7 +309,7 @@ class DbmCacheStorage:
|
|||
if 0 < self.expiration_secs < time() - float(ts):
|
||||
return None # expired
|
||||
|
||||
return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec
|
||||
return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # noqa: S301
|
||||
|
||||
|
||||
class FilesystemCacheStorage:
|
||||
|
|
@ -392,7 +392,7 @@ class FilesystemCacheStorage:
|
|||
if 0 < self.expiration_secs < time() - mtime:
|
||||
return None # expired
|
||||
with self._open(metapath, "rb") as f:
|
||||
return cast(dict[str, Any], pickle.load(f)) # nosec
|
||||
return cast(dict[str, Any], pickle.load(f)) # noqa: S301
|
||||
|
||||
|
||||
def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pickle # nosec
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
|
|
@ -41,7 +41,7 @@ class SpiderState:
|
|||
def spider_opened(self, spider: Spider) -> None:
|
||||
if self.jobdir and Path(self.statefn).exists():
|
||||
with Path(self.statefn).open("rb") as f:
|
||||
spider.state = pickle.load(f) # type: ignore[attr-defined] # nosec
|
||||
spider.state = pickle.load(f) # type: ignore[attr-defined] # noqa: S301
|
||||
else:
|
||||
spider.state = {} # type: ignore[attr-defined]
|
||||
|
||||
|
|
|
|||
|
|
@ -11,11 +11,13 @@ from collections.abc import Iterable
|
|||
from typing import TYPE_CHECKING, Any, Optional, Union, cast
|
||||
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
|
||||
|
||||
from lxml.html import FormElement # nosec
|
||||
from lxml.html import InputElement # nosec
|
||||
from lxml.html import MultipleSelectOptions # nosec
|
||||
from lxml.html import SelectElement # nosec
|
||||
from lxml.html import TextareaElement # nosec
|
||||
from lxml.html import (
|
||||
FormElement,
|
||||
InputElement,
|
||||
MultipleSelectOptions,
|
||||
SelectElement,
|
||||
TextareaElement,
|
||||
)
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.http.request import Request
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ class Response(object_ref):
|
|||
self._url: str = url
|
||||
else:
|
||||
raise TypeError(
|
||||
f"{type(self).__name__} url must be str, " f"got {type(url).__name__}"
|
||||
f"{type(self).__name__} url must be str, got {type(url).__name__}"
|
||||
)
|
||||
|
||||
@property
|
||||
|
|
|
|||
|
|
@ -308,7 +308,7 @@ def _url_from_selector(sel: parsel.Selector) -> str:
|
|||
raise _InvalidSelector(f"Unsupported selector: {sel}")
|
||||
if sel.root.tag not in ("a", "link"):
|
||||
raise _InvalidSelector(
|
||||
"Only <a> and <link> elements are supported; " f"got <{sel.root.tag}>"
|
||||
f"Only <a> and <link> elements are supported; got <{sel.root.tag}>"
|
||||
)
|
||||
href = sel.root.get("href")
|
||||
if href is None:
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from functools import partial
|
|||
from typing import TYPE_CHECKING, Any, Union, cast
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from lxml import etree # nosec
|
||||
from lxml import etree
|
||||
from parsel.csstranslator import HTMLTranslator
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
from w3lib.url import canonicalize_url, safe_url_string
|
||||
|
|
@ -26,7 +26,7 @@ from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain
|
|||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
from lxml.html import HtmlElement # nosec
|
||||
from lxml.html import HtmlElement
|
||||
|
||||
from scrapy import Selector
|
||||
from scrapy.http import TextResponse
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ def _md5sum(file: IO[bytes]) -> str:
|
|||
>>> _md5sum(BytesIO(b'file content to hash'))
|
||||
'784406af91dd5a54fbb9c84c2236595a'
|
||||
"""
|
||||
m = hashlib.md5() # nosec
|
||||
m = hashlib.md5() # noqa: S324
|
||||
while True:
|
||||
d = file.read(8096)
|
||||
if not d:
|
||||
|
|
@ -399,7 +399,7 @@ class FTPFilesStore:
|
|||
ftp.set_pasv(False)
|
||||
file_path = f"{self.basedir}/{path}"
|
||||
last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())
|
||||
m = hashlib.md5() # nosec
|
||||
m = hashlib.md5() # noqa: S324
|
||||
ftp.retrbinary(f"RETR {file_path}", m.update)
|
||||
return {"last_modified": last_modified, "checksum": m.hexdigest()}
|
||||
# The file doesn't exist
|
||||
|
|
@ -734,7 +734,7 @@ class FilesPipeline(MediaPipeline):
|
|||
*,
|
||||
item: Any = None,
|
||||
) -> str:
|
||||
media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec
|
||||
media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324
|
||||
media_ext = Path(request.url).suffix
|
||||
# Handles empty and wild extensions by trying to guess the
|
||||
# mime type then extension or default to empty string otherwise
|
||||
|
|
|
|||
|
|
@ -258,7 +258,7 @@ class ImagesPipeline(FilesPipeline):
|
|||
*,
|
||||
item: Any = None,
|
||||
) -> str:
|
||||
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec
|
||||
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324
|
||||
return f"full/{image_guid}.jpg"
|
||||
|
||||
def thumb_path(
|
||||
|
|
@ -270,5 +270,5 @@ class ImagesPipeline(FilesPipeline):
|
|||
*,
|
||||
item: Any = None,
|
||||
) -> str:
|
||||
thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec
|
||||
thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324
|
||||
return f"thumbs/{thumb_id}/{thumb_guid}.jpg"
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ def _path_safe(text: str) -> str:
|
|||
pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text])
|
||||
# as we replace some letters we can get collision for different slots
|
||||
# add we add unique part
|
||||
unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # nosec
|
||||
unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # noqa: S324
|
||||
return "-".join([pathable_slot, unique_slot])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ FILES_STORE_S3_ACL = "private"
|
|||
FILES_STORE_GCS_ACL = ""
|
||||
|
||||
FTP_USER = "anonymous"
|
||||
FTP_PASSWORD = "guest" # nosec
|
||||
FTP_PASSWORD = "guest" # noqa: S105
|
||||
FTP_PASSIVE_MODE = True
|
||||
|
||||
GCS_PROJECT_ID = None
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ class Shell:
|
|||
else:
|
||||
self.populate_vars()
|
||||
if self.code:
|
||||
print(eval(self.code, globals(), self.vars)) # nosec
|
||||
print(eval(self.code, globals(), self.vars)) # noqa: S307
|
||||
else:
|
||||
"""
|
||||
Detect interactive shell setting in scrapy.cfg
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ Scheduler queues
|
|||
from __future__ import annotations
|
||||
|
||||
import marshal
|
||||
import pickle # nosec
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class Root(Resource):
|
|||
def render(self, request: Request) -> bytes:
|
||||
total = _getarg(request, b"total", 100, int)
|
||||
show = _getarg(request, b"show", 10, int)
|
||||
nlist = [random.randint(1, total) for _ in range(show)] # nosec
|
||||
nlist = [random.randint(1, total) for _ in range(show)] # noqa: S311
|
||||
request.write(b"<html><head></head><body>")
|
||||
assert request.args is not None
|
||||
args = request.args.copy()
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]:
|
|||
checks: list[tuple[str, Any]] = []
|
||||
for test in tests:
|
||||
try:
|
||||
checks += [(test, eval(test))] # nosec
|
||||
checks += [(test, eval(test))] # noqa: S307
|
||||
except Exception as e:
|
||||
checks += [(test, f"{type(e).__name__} (exception)")]
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from io import StringIO
|
|||
from typing import TYPE_CHECKING, Any, Literal, cast, overload
|
||||
from warnings import warn
|
||||
|
||||
from lxml import etree # nosec
|
||||
from lxml import etree
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Response, TextResponse
|
||||
|
|
@ -41,10 +41,10 @@ def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]:
|
|||
|
||||
nodename_patt = re.escape(nodename)
|
||||
|
||||
DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S)
|
||||
HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S)
|
||||
END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S)
|
||||
NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S)
|
||||
DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.DOTALL)
|
||||
HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.DOTALL)
|
||||
END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.DOTALL)
|
||||
NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.DOTALL)
|
||||
text = _body_or_str(obj)
|
||||
|
||||
document_header_match = re.search(DOCUMENT_HEADER_RE, text)
|
||||
|
|
@ -58,7 +58,9 @@ def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]:
|
|||
for tagname in reversed(re.findall(END_TAG_RE, header_end)):
|
||||
assert header_end_idx
|
||||
tag = re.search(
|
||||
rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S
|
||||
rf"<\s*{tagname}.*?xmlns[:=][^>]*>",
|
||||
text[: header_end_idx[1]],
|
||||
re.DOTALL,
|
||||
)
|
||||
if tag:
|
||||
for x in re.findall(NAMESPACE_RE, tag.group()):
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ def md5sum(file: IO[bytes]) -> str:
|
|||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
m = hashlib.md5() # nosec
|
||||
m = hashlib.md5() # noqa: S324
|
||||
while True:
|
||||
d = file.read(8096)
|
||||
if not d:
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ def to_bytes(
|
|||
return text
|
||||
if not isinstance(text, str):
|
||||
raise TypeError(
|
||||
"to_bytes must receive a str or bytes " f"object, got {type(text).__name__}"
|
||||
f"to_bytes must receive a str or bytes object, got {type(text).__name__}"
|
||||
)
|
||||
if encoding is None:
|
||||
encoding = "utf-8"
|
||||
|
|
|
|||
|
|
@ -94,7 +94,9 @@ def fingerprint(
|
|||
"headers": headers,
|
||||
}
|
||||
fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
|
||||
cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() # nosec
|
||||
cache[cache_key] = hashlib.sha1( # noqa: S324
|
||||
fingerprint_json.encode()
|
||||
).digest()
|
||||
return cache[cache_key]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ def open_in_browser(
|
|||
elif isinstance(response, TextResponse):
|
||||
ext = ".txt"
|
||||
else:
|
||||
raise TypeError("Unsupported response type: " f"{response.__class__.__name__}")
|
||||
raise TypeError(f"Unsupported response type: {response.__class__.__name__}")
|
||||
fd, fname = tempfile.mkstemp(ext)
|
||||
os.write(fd, body)
|
||||
os.close(fd)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from __future__ import annotations
|
|||
from typing import TYPE_CHECKING, Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import lxml.etree # nosec
|
||||
import lxml.etree
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Iterator
|
||||
|
|
@ -24,7 +24,7 @@ class Sitemap:
|
|||
xmlp = lxml.etree.XMLParser(
|
||||
recover=True, remove_comments=True, resolve_entities=False
|
||||
)
|
||||
self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # nosec
|
||||
self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # noqa: S320
|
||||
rt = self._root.tag
|
||||
self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt
|
||||
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ def escape_ajax(url: str) -> str:
|
|||
|
||||
def add_http_if_no_scheme(url: str) -> str:
|
||||
"""Add http as the default scheme if it is missing from the url."""
|
||||
match = re.match(r"^\w+://", url, flags=re.I)
|
||||
match = re.match(r"^\w+://", url, flags=re.IGNORECASE)
|
||||
if not match:
|
||||
parts = urlparse(url)
|
||||
scheme = "http:" if parts.netloc else "http://"
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import sys
|
|||
|
||||
import cryptography
|
||||
import cssselect
|
||||
import lxml.etree # nosec
|
||||
import lxml.etree
|
||||
import parsel
|
||||
import twisted
|
||||
import w3lib
|
||||
|
|
|
|||
|
|
@ -238,7 +238,7 @@ class StartprojectTemplatesTest(ProjectTest):
|
|||
args = ["--set", f"TEMPLATES_DIR={self.tmpl}"]
|
||||
p, out, err = self.proc("startproject", self.project_name, *args)
|
||||
self.assertIn(
|
||||
f"New Scrapy project '{self.project_name}', " "using template directory",
|
||||
f"New Scrapy project '{self.project_name}', using template directory",
|
||||
out,
|
||||
)
|
||||
self.assertIn(self.tmpl_proj, out)
|
||||
|
|
|
|||
|
|
@ -67,8 +67,8 @@ class TestSpider(Spider):
|
|||
allowed_domains = ["scrapytest.org", "localhost"]
|
||||
|
||||
itemurl_re = re.compile(r"item\d+.html")
|
||||
name_re = re.compile(r"<h1>(.*?)</h1>", re.M)
|
||||
price_re = re.compile(r">Price: \$(.*?)<", re.M)
|
||||
name_re = re.compile(r"<h1>(.*?)</h1>", re.MULTILINE)
|
||||
price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE)
|
||||
|
||||
item_cls: type = TestItem
|
||||
|
||||
|
|
|
|||
|
|
@ -631,7 +631,7 @@ class TestGCSFilesStore(unittest.TestCase):
|
|||
"""
|
||||
assert_gcs_environ()
|
||||
try:
|
||||
import google.cloud.storage # noqa
|
||||
import google.cloud.storage # noqa: F401
|
||||
except ModuleNotFoundError:
|
||||
raise unittest.SkipTest("google-cloud-storage is not installed")
|
||||
with mock.patch("google.cloud.storage") as _:
|
||||
|
|
|
|||
|
|
@ -66,14 +66,14 @@ class BaseRobotParserTest:
|
|||
self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second"))
|
||||
|
||||
def test_length_based_precedence(self):
|
||||
robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"
|
||||
robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"
|
||||
rp = self.parser_cls.from_crawler(
|
||||
crawler=None, robotstxt_body=robotstxt_robotstxt_body
|
||||
)
|
||||
self.assertTrue(rp.allowed("https://www.site.local/page", "*"))
|
||||
|
||||
def test_order_based_precedence(self):
|
||||
robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"
|
||||
robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"
|
||||
rp = self.parser_cls.from_crawler(
|
||||
crawler=None, robotstxt_body=robotstxt_robotstxt_body
|
||||
)
|
||||
|
|
|
|||
|
|
@ -21,9 +21,9 @@ class TrackrefTestCase(unittest.TestCase):
|
|||
trackref.live_refs.clear()
|
||||
|
||||
def test_format_live_refs(self):
|
||||
o1 = Foo() # NOQA
|
||||
o2 = Bar() # NOQA
|
||||
o3 = Foo() # NOQA
|
||||
o1 = Foo() # noqa: F841
|
||||
o2 = Bar() # noqa: F841
|
||||
o3 = Foo() # noqa: F841
|
||||
self.assertEqual(
|
||||
trackref.format_live_refs(),
|
||||
"""\
|
||||
|
|
@ -50,7 +50,7 @@ Bar 1 oldest: 0s ago
|
|||
|
||||
@mock.patch("sys.stdout", new_callable=StringIO)
|
||||
def test_print_live_refs_with_objects(self, stdout):
|
||||
o1 = Foo() # NOQA
|
||||
o1 = Foo() # noqa: F841
|
||||
trackref.print_live_refs()
|
||||
self.assertEqual(
|
||||
stdout.getvalue(),
|
||||
|
|
@ -61,11 +61,11 @@ Foo 1 oldest: 0s ago\n\n""",
|
|||
)
|
||||
|
||||
def test_get_oldest(self):
|
||||
o1 = Foo() # NOQA
|
||||
o1 = Foo() # noqa: F841
|
||||
|
||||
o1_time = time()
|
||||
|
||||
o2 = Bar() # NOQA
|
||||
o2 = Bar() # noqa: F841
|
||||
|
||||
o3_time = time()
|
||||
if o3_time <= o1_time:
|
||||
|
|
@ -74,15 +74,15 @@ Foo 1 oldest: 0s ago\n\n""",
|
|||
if o3_time <= o1_time:
|
||||
raise SkipTest("time.time is not precise enough")
|
||||
|
||||
o3 = Foo() # NOQA
|
||||
o3 = Foo() # noqa: F841
|
||||
self.assertIs(trackref.get_oldest("Foo"), o1)
|
||||
self.assertIs(trackref.get_oldest("Bar"), o2)
|
||||
self.assertIsNone(trackref.get_oldest("XXX"))
|
||||
|
||||
def test_iter_all(self):
|
||||
o1 = Foo() # NOQA
|
||||
o2 = Bar() # NOQA
|
||||
o3 = Foo() # NOQA
|
||||
o1 = Foo() # noqa: F841
|
||||
o2 = Bar() # noqa: F841
|
||||
o3 = Foo() # noqa: F841
|
||||
self.assertEqual(
|
||||
set(trackref.iter_all("Foo")),
|
||||
{o1, o3},
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
|
||||
# test minimal sent headers
|
||||
factory = client.ScrapyHTTPClientFactory(Request("http://foo/bar"))
|
||||
self._test(factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"\r\n")
|
||||
self._test(factory, b"GET /bar HTTP/1.0\r\nHost: foo\r\n\r\n")
|
||||
|
||||
# test a simple POST with body and content-type
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
|
|
@ -191,7 +191,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
|
||||
self._test(
|
||||
factory,
|
||||
b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Content-Length: 0\r\n" b"\r\n",
|
||||
b"POST /bar HTTP/1.0\r\nHost: foo\r\nContent-Length: 0\r\n\r\n",
|
||||
)
|
||||
|
||||
# test with single and multivalued headers
|
||||
|
|
|
|||
Loading…
Reference in New Issue