Merge pull request #2107 from redapple/backport-1.1-pr2052

[backport][1.1] Enable genspider command outside project folder (PR #2052)

docs/topics/commands.rst

@@ -159,6 +159,7 @@ settings).
 Global commands:
 
 * :command:`startproject`
+* :command:`genspider`
 * :command:`settings`
 * :command:`runspider`
 * :command:`shell`
@@ -173,7 +174,6 @@ Project-only commands:
 * :command:`list`
 * :command:`edit`
 * :command:`parse`
-* :command:`genspider`
 * :command:`bench`
 
 .. command:: startproject
@@ -197,14 +197,9 @@ genspider
 ---------
 
 * Syntax: ``scrapy genspider [-t template] <name> <domain>``
-* Requires project: *yes*
+* Requires project: *no*
 
-Create a new spider in the current project.
-
-This is just a convenience shortcut command for creating spiders based on
-pre-defined templates, but certainly not the only way to create spiders. You
-can just create the spider source code files yourself, instead of using this
-command.
+Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
 
 Usage example::
 
@@ -215,22 +210,16 @@ Usage example::
       csvfeed
       xmlfeed
 
-    $ scrapy genspider -d basic
-    import scrapy
+    $ scrapy genspider example example.com
+    Created spider 'example' using template 'basic'
 
-    class $classname(scrapy.Spider):
-        name = "$name"
-        allowed_domains = ["$domain"]
-        start_urls = (
-            'http://www.$domain/',
-            )
+    $ scrapy genspider -t crawl scrapyorg scrapy.org
+    Created spider 'scrapyorg' using template 'crawl'
 
-        def parse(self, response):
-            pass
-
-    $ scrapy genspider -t basic example example.com
-    Created spider 'example' using template 'basic' in module:
-      mybot.spiders.example
+This is just a convenience shortcut command for creating spiders based on
+pre-defined templates, but certainly not the only way to create spiders. You
+can just create the spider source code files yourself, instead of using this
+command.
 
 .. command:: crawl
 

scrapy/commands/genspider.py

@@ -25,7 +25,7 @@ def sanitize_module_name(module_name):
 
 class Command(ScrapyCommand):
 
-    requires_project = True
+    requires_project = False
     default_settings = {'LOG_ENABLED': False}
 
     def syntax(self):
@@ -94,14 +94,19 @@ class Command(ScrapyCommand):
             'classname': '%sSpider' % ''.join(s.capitalize() \
                 for s in module.split('_'))
         }
-        spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
-        spiders_dir = abspath(dirname(spiders_module.__file__))
+        if self.settings.get('NEWSPIDER_MODULE'):
+            spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
+            spiders_dir = abspath(dirname(spiders_module.__file__))
+        else:
+            spiders_module = None
+            spiders_dir = "."
         spider_file = "%s.py" % join(spiders_dir, module)
         shutil.copyfile(template_file, spider_file)
         render_templatefile(spider_file, **tvars)
-        print("Created spider %r using template %r in module:" % (name, \
-            template_name))
-        print("  %s.%s" % (spiders_module.__name__, module))
+        print("Created spider %r using template %r " % (name, \
+            template_name), end=('' if spiders_module else '\n'))
+        if spiders_module:
+            print("in module:\n  %s.%s" % (spiders_module.__name__, module))
 
     def _find_template(self, template):
         template_file = join(self.templates_dir, '%s.tmpl' % template)

scrapy/templates/spiders/crawl.tmpl

@@ -3,8 +3,6 @@ import scrapy
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule
 
-from $project_name.items import ${ProjectName}Item
-
 
 class $classname(CrawlSpider):
     name = '$name'
@@ -16,7 +14,7 @@ class $classname(CrawlSpider):
     )
 
     def parse_item(self, response):
-        i = ${ProjectName}Item()
+        i = {}
         #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
         #i['name'] = response.xpath('//div[@id="name"]').extract()
         #i['description'] = response.xpath('//div[@id="description"]').extract()

scrapy/templates/spiders/csvfeed.tmpl

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 from scrapy.spiders import CSVFeedSpider
 
-from $project_name.items import ${ProjectName}Item
-
 
 class $classname(CSVFeedSpider):
     name = '$name'
@@ -16,7 +14,7 @@ class $classname(CSVFeedSpider):
     #    return response
 
     def parse_row(self, response, row):
-        i = ${ProjectName}Item()
+        i = {}
         #i['url'] = row['url']
         #i['name'] = row['name']
         #i['description'] = row['description']

scrapy/templates/spiders/xmlfeed.tmpl

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 from scrapy.spiders import XMLFeedSpider
 
-from $project_name.items import ${ProjectName}Item
-
 
 class $classname(XMLFeedSpider):
     name = '$name'
@@ -12,7 +10,7 @@ class $classname(XMLFeedSpider):
     itertag = 'item' # change it accordingly
 
     def parse_node(self, response, selector):
-        i = ${ProjectName}Item()
+        i = {}
         #i['url'] = selector.select('url').extract()
         #i['name'] = selector.select('name').extract()
         #i['description'] = selector.select('description').extract()

tests/test_commands.py

@@ -146,6 +146,13 @@ class GenspiderCommandTest(CommandTest):
         assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name))
 
 
+class GenspiderStandaloneCommandTest(ProjectTest):
+
+    def test_generate_standalone_spider(self):
+        self.call('genspider', 'example', 'example.com')
+        assert exists(join(self.temp_path, 'example.py'))
+
+
 class MiscCommandsTest(CommandTest):
 
     def test_list(self):