Refactor genspider slightly so template variables can be overridden (#6470)

This commit is contained in:
Daniel O'Connor 2024-08-29 04:37:49 +09:30 committed by GitHub
parent c9d85faaf2
commit 67ab8d4650
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 19 additions and 11 deletions

View File

@ -116,6 +116,24 @@ class Command(ScrapyCommand):
if opts.edit:
self.exitcode = os.system(f'scrapy edit "{name}"') # nosec
def _generate_template_variables(
self,
module: str,
name: str,
url: str,
template_name: str,
):
capitalized_module = "".join(s.capitalize() for s in module.split("_"))
return {
"project_name": self.settings.get("BOT_NAME"),
"ProjectName": string_camelcase(self.settings.get("BOT_NAME")),
"module": module,
"name": name,
"url": url,
"domain": extract_domain(url),
"classname": f"{capitalized_module}Spider",
}
def _genspider(
self,
module: str,
@ -125,17 +143,7 @@ class Command(ScrapyCommand):
template_file: Union[str, os.PathLike],
) -> None:
"""Generate the spider module, based on the given template"""
capitalized_module = "".join(s.capitalize() for s in module.split("_"))
domain = extract_domain(url)
tvars = {
"project_name": self.settings.get("BOT_NAME"),
"ProjectName": string_camelcase(self.settings.get("BOT_NAME")),
"module": module,
"name": name,
"url": url,
"domain": domain,
"classname": f"{capitalized_module}Spider",
}
tvars = self._generate_template_variables(module, name, url, template_name)
if self.settings.get("NEWSPIDER_MODULE"):
spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
assert spiders_module.__file__