diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index bd09890fe..8c9a86dce 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -53,7 +53,7 @@ class ${ProjectName}SpiderMiddleware: yield r def spider_opened(self, spider): - spider.logger.info('Spider opened: %s' % spider.name) + spider.logger.info("Spider opened: %s" % spider.name) class ${ProjectName}DownloaderMiddleware: @@ -100,4 +100,4 @@ class ${ProjectName}DownloaderMiddleware: pass def spider_opened(self, spider): - spider.logger.info('Spider opened: %s' % spider.name) + spider.logger.info("Spider opened: %s" % spider.name) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 2f6df5abc..ecb1e5e5c 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -7,14 +7,14 @@ # https://docs.scrapy.org/en/latest/topics/downloader-middleware.html # https://docs.scrapy.org/en/latest/topics/spider-middleware.html -BOT_NAME = '$project_name' +BOT_NAME = "$project_name" -SPIDER_MODULES = ['$project_name.spiders'] -NEWSPIDER_MODULE = '$project_name.spiders' +SPIDER_MODULES = ["$project_name.spiders"] +NEWSPIDER_MODULE = "$project_name.spiders" # Crawl responsibly by identifying yourself (and your website) on the user-agent -#USER_AGENT = '$project_name (+http://www.yourdomain.com)' +#USER_AGENT = "$project_name (+http://www.yourdomain.com)" # Obey robots.txt rules ROBOTSTXT_OBEY = True @@ -38,32 +38,32 @@ ROBOTSTXT_OBEY = True # Override the default request headers: #DEFAULT_REQUEST_HEADERS = { -# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', -# 'Accept-Language': 'en', +# "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", +# "Accept-Language": "en", #} # Enable or disable spider middlewares # See https://docs.scrapy.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { -# '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, +# "$project_name.middlewares.${ProjectName}SpiderMiddleware": 543, #} # Enable or disable downloader middlewares # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { -# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, +# "$project_name.middlewares.${ProjectName}DownloaderMiddleware": 543, #} # Enable or disable extensions # See https://docs.scrapy.org/en/latest/topics/extensions.html #EXTENSIONS = { -# 'scrapy.extensions.telnet.TelnetConsole': None, +# "scrapy.extensions.telnet.TelnetConsole": None, #} # Configure item pipelines # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html #ITEM_PIPELINES = { -# '$project_name.pipelines.${ProjectName}Pipeline': 300, +# "$project_name.pipelines.${ProjectName}Pipeline": 300, #} # Enable and configure the AutoThrottle extension (disabled by default) @@ -83,11 +83,11 @@ ROBOTSTXT_OBEY = True # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 -#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_DIR = "httpcache" #HTTPCACHE_IGNORE_HTTP_CODES = [] -#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' +#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" # Set settings whose default value is deprecated to a future-proof value -REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7' -TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' -FEED_EXPORT_ENCODING = 'utf-8' +REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" +TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" +FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index e9112bc95..d3ba19553 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -2,9 +2,9 @@ import scrapy class $classname(scrapy.Spider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/'] + name = "$name" + allowed_domains = ["$domain"] + start_urls = ["http://$domain/"] def parse(self, response): pass diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 356496487..2e467e632 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -4,17 +4,15 @@ from scrapy.spiders import CrawlSpider, Rule class $classname(CrawlSpider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/'] + name = "$name" + allowed_domains = ["$domain"] + start_urls = ["http://$domain/"] - rules = ( - Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), - ) + rules = (Rule(LinkExtractor(allow=r"Items/"), callback="parse_item", follow=True),) def parse_item(self, response): item = {} - #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get() - #item['name'] = response.xpath('//div[@id="name"]').get() - #item['description'] = response.xpath('//div[@id="description"]').get() + #item["domain_id"] = response.xpath('//input[@id="sid"]/@value').get() + #item["name"] = response.xpath('//div[@id="name"]').get() + #item["description"] = response.xpath('//div[@id="description"]').get() return item diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index cbcbe9e2c..ce9c1dd20 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -2,11 +2,11 @@ from scrapy.spiders import CSVFeedSpider class $classname(CSVFeedSpider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/feed.csv'] - # headers = ['id', 'name', 'description', 'image_link'] - # delimiter = '\t' + name = "$name" + allowed_domains = ["$domain"] + start_urls = ["http://$domain/feed.csv"] + #headers = ["id", "name", "description", "image_link"] + #delimiter = "\t" # Do any adaptations you need here #def adapt_response(self, response): @@ -14,7 +14,7 @@ class $classname(CSVFeedSpider): def parse_row(self, response, row): i = {} - #i['url'] = row['url'] - #i['name'] = row['name'] - #i['description'] = row['description'] + #i["url"] = row["url"] + #i["name"] = row["name"] + #i["description"] = row["description"] return i diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 5aa2aa8b0..6b50e4cf4 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -2,15 +2,15 @@ from scrapy.spiders import XMLFeedSpider class $classname(XMLFeedSpider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/feed.xml'] - iterator = 'iternodes' # you can change this; see the docs - itertag = 'item' # change it accordingly + name = "$name" + allowed_domains = ["$domain"] + start_urls = ["http://$domain/feed.xml"] + iterator = "iternodes" # you can change this; see the docs + itertag = "item" # change it accordingly def parse_node(self, response, selector): item = {} - #item['url'] = selector.select('url').get() - #item['name'] = selector.select('name').get() - #item['description'] = selector.select('description').get() + #item["url"] = selector.select("url").get() + #item["name"] = selector.select("name").get() + #item["description"] = selector.select("description").get() return item diff --git a/tests/test_commands.py b/tests/test_commands.py index 5ff2dd482..d7ee75559 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -505,7 +505,7 @@ class GenspiderCommandTest(CommandTest): # change name of spider but not its file name with file_path.open("r+", encoding="utf-8") as spider_file: file_data = spider_file.read() - file_data = file_data.replace("name = 'example'", "name = 'renamed'") + file_data = file_data.replace('name = "example"', 'name = "renamed"') spider_file.seek(0) spider_file.write(file_data) spider_file.truncate() @@ -538,14 +538,14 @@ class GenspiderCommandTest(CommandTest): domain, self.find_in_file( Path(self.proj_mod_path, "spiders", "test_name.py"), - r"allowed_domains\s*=\s*\[\'(.+)\'\]", + r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]", ).group(1), ) self.assertEqual( f"http://{domain}/", self.find_in_file( Path(self.proj_mod_path, "spiders", "test_name.py"), - r"start_urls\s*=\s*\[\'(.+)\'\]", + r"start_urls\s*=\s*\[['\"](.+)['\"]\]", ).group(1), )