From 67ab8d4650c1e9212c9508803c7b5265e166cbaa Mon Sep 17 00:00:00 2001 From: Daniel O'Connor Date: Thu, 29 Aug 2024 04:37:49 +0930 Subject: [PATCH] Refactor genspider slightly so template variables can be overridden (#6470) --- scrapy/commands/genspider.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2649fb23d..6c3713f8f 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -116,6 +116,24 @@ class Command(ScrapyCommand): if opts.edit: self.exitcode = os.system(f'scrapy edit "{name}"') # nosec + def _generate_template_variables( + self, + module: str, + name: str, + url: str, + template_name: str, + ): + capitalized_module = "".join(s.capitalize() for s in module.split("_")) + return { + "project_name": self.settings.get("BOT_NAME"), + "ProjectName": string_camelcase(self.settings.get("BOT_NAME")), + "module": module, + "name": name, + "url": url, + "domain": extract_domain(url), + "classname": f"{capitalized_module}Spider", + } + def _genspider( self, module: str, @@ -125,17 +143,7 @@ class Command(ScrapyCommand): template_file: Union[str, os.PathLike], ) -> None: """Generate the spider module, based on the given template""" - capitalized_module = "".join(s.capitalize() for s in module.split("_")) - domain = extract_domain(url) - tvars = { - "project_name": self.settings.get("BOT_NAME"), - "ProjectName": string_camelcase(self.settings.get("BOT_NAME")), - "module": module, - "name": name, - "url": url, - "domain": domain, - "classname": f"{capitalized_module}Spider", - } + tvars = self._generate_template_variables(module, name, url, template_name) if self.settings.get("NEWSPIDER_MODULE"): spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) assert spiders_module.__file__