From acb0c325aac678e071de69e7f9715c82203c90de Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Sat, 11 Jun 2016 20:44:08 -0300 Subject: [PATCH 1/3] enable genspider command outside projects --- scrapy/commands/genspider.py | 17 +++++++++++------ tests/test_commands.py | 7 +++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 58bdb9156..d5498bb5c 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -25,7 +25,7 @@ def sanitize_module_name(module_name): class Command(ScrapyCommand): - requires_project = True + requires_project = False default_settings = {'LOG_ENABLED': False} def syntax(self): @@ -94,14 +94,19 @@ class Command(ScrapyCommand): 'classname': '%sSpider' % ''.join(s.capitalize() \ for s in module.split('_')) } - spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) - spiders_dir = abspath(dirname(spiders_module.__file__)) + if self.settings.get('NEWSPIDER_MODULE'): + spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) + spiders_dir = abspath(dirname(spiders_module.__file__)) + else: + spiders_module = None + spiders_dir = "." spider_file = "%s.py" % join(spiders_dir, module) shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) - print("Created spider %r using template %r in module:" % (name, \ - template_name)) - print(" %s.%s" % (spiders_module.__name__, module)) + print("Created spider %r using template %r " % (name, \ + template_name), end=('' if spiders_module else '\n')) + if spiders_module: + print("in module:\n %s.%s" % (spiders_module.__name__, module)) def _find_template(self, template): template_file = join(self.templates_dir, '%s.tmpl' % template) diff --git a/tests/test_commands.py b/tests/test_commands.py index 2e47160d7..cf415a388 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -146,6 +146,13 @@ class GenspiderCommandTest(CommandTest): assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name)) +class GenspiderStandaloneCommandTest(ProjectTest): + + def test_generate_standalone_spider(self): + self.call('genspider', 'example', 'example.com') + assert exists(join(self.temp_path, 'example.py')) + + class MiscCommandsTest(CommandTest): def test_list(self): From e239246f4bc9d0213d7dda8ae86bba13a1dec0b7 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Mon, 4 Jul 2016 18:59:53 -0300 Subject: [PATCH 2/3] remove references to Item classes in templates --- scrapy/templates/spiders/crawl.tmpl | 4 +--- scrapy/templates/spiders/csvfeed.tmpl | 4 +--- scrapy/templates/spiders/xmlfeed.tmpl | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index a179d16ff..154237d9c 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -3,8 +3,6 @@ import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule -from $project_name.items import ${ProjectName}Item - class $classname(CrawlSpider): name = '$name' @@ -16,7 +14,7 @@ class $classname(CrawlSpider): ) def parse_item(self, response): - i = ${ProjectName}Item() + i = {} #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract() #i['name'] = response.xpath('//div[@id="name"]').extract() #i['description'] = response.xpath('//div[@id="description"]').extract() diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index 69c606538..0544e0ae7 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- from scrapy.spiders import CSVFeedSpider -from $project_name.items import ${ProjectName}Item - class $classname(CSVFeedSpider): name = '$name' @@ -16,7 +14,7 @@ class $classname(CSVFeedSpider): # return response def parse_row(self, response, row): - i = ${ProjectName}Item() + i = {} #i['url'] = row['url'] #i['name'] = row['name'] #i['description'] = row['description'] diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 9c0910d23..d8ff61f6e 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- from scrapy.spiders import XMLFeedSpider -from $project_name.items import ${ProjectName}Item - class $classname(XMLFeedSpider): name = '$name' @@ -12,7 +10,7 @@ class $classname(XMLFeedSpider): itertag = 'item' # change it accordingly def parse_node(self, response, selector): - i = ${ProjectName}Item() + i = {} #i['url'] = selector.select('url').extract() #i['name'] = selector.select('name').extract() #i['description'] = selector.select('description').extract() From 848bd4c2a24f945a55a3d4def44b8443dd23a9c1 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Tue, 5 Jul 2016 22:48:18 -0300 Subject: [PATCH 3/3] document new genspider behavior --- docs/topics/commands.rst | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 9a40a2c29..d7999900b 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -159,6 +159,7 @@ settings). Global commands: * :command:`startproject` +* :command:`genspider` * :command:`settings` * :command:`runspider` * :command:`shell` @@ -173,7 +174,6 @@ Project-only commands: * :command:`list` * :command:`edit` * :command:`parse` -* :command:`genspider` * :command:`bench` .. command:: startproject @@ -197,14 +197,9 @@ genspider --------- * Syntax: ``scrapy genspider [-t template] `` -* Requires project: *yes* +* Requires project: *no* -Create a new spider in the current project. - -This is just a convenience shortcut command for creating spiders based on -pre-defined templates, but certainly not the only way to create spiders. You -can just create the spider source code files yourself, instead of using this -command. +Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. Usage example:: @@ -215,22 +210,16 @@ Usage example:: csvfeed xmlfeed - $ scrapy genspider -d basic - import scrapy + $ scrapy genspider example example.com + Created spider 'example' using template 'basic' - class $classname(scrapy.Spider): - name = "$name" - allowed_domains = ["$domain"] - start_urls = ( - 'http://www.$domain/', - ) + $ scrapy genspider -t crawl scrapyorg scrapy.org + Created spider 'scrapyorg' using template 'crawl' - def parse(self, response): - pass - - $ scrapy genspider -t basic example example.com - Created spider 'example' using template 'basic' in module: - mybot.spiders.example +This is just a convenience shortcut command for creating spiders based on +pre-defined templates, but certainly not the only way to create spiders. You +can just create the spider source code files yourself, instead of using this +command. .. command:: crawl