Fix usage of crawler_process in ScrapyCommands

scrapy/commands/bench.py

@@ -22,9 +22,7 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         with _BenchServer():
-            spider = _BenchSpider(total=100000)
-            crawler = self.crawler_process.create_crawler()
-            crawler.crawl(spider)
+            self.crawler_process.crawl(_BenchSpider, total=100000)
             self.crawler_process.start()
 
 

scrapy/commands/check.py

@@ -69,20 +69,18 @@ class Command(ScrapyCommand):
         # contract requests
         contract_reqs = defaultdict(list)
 
-        spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
-        spiders = spman_cls.from_settings(self.settings)
+        spiders = self.crawler_process.spiders
 
-        for spider in args or spiders.list():
-            spider = spiders.create(spider)
-            requests = self.get_requests(spider, conman, result)
-            contract_reqs[spider.name] = []
+        for spidername in args or spiders.list():
+            spidercls = spiders.load(spidername)
+            spidercls.start_requests = lambda s: conman.from_spider(s, result)
 
+            tested_methods = conman.tested_methods_from_spidercls(spidercls)
             if opts.list:
-                for req in requests:
-                    contract_reqs[spider.name].append(req.callback.__name__)
-            elif requests:
-                crawler = self.crawler_process.create_crawler(spider.name)
-                crawler.crawl(spider, requests)
+                for method in tested_methods:
+                    contract_reqs[spidercls.name].append(method)
+            elif tested_methods:
+                self.crawler_process.crawl(spidercls)
 
         # start checks
         if opts.list:
@@ -101,15 +99,3 @@ class Command(ScrapyCommand):
             result.printSummary(start, stop)
             self.exitcode = int(not result.wasSuccessful())
 
-    def get_requests(self, spider, conman, result):
-        requests = []
-
-        for key, value in vars(type(spider)).items():
-            if callable(value) and value.__doc__:
-                bound_method = value.__get__(spider, type(spider))
-                request = conman.from_method(bound_method, result)
-
-                if request:
-                    requests.append(request)
-
-        return requests

scrapy/commands/crawl.py

@@ -54,7 +54,5 @@ class Command(ScrapyCommand):
             raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
         spname = args[0]
 
-        crawler = self.crawler_process.create_crawler()
-        spider = crawler.spiders.create(spname, **opts.spargs)
-        crawler.crawl(spider)
+        self.crawler_process.crawl(spname, **opts.spargs)
         self.crawler_process.start()

scrapy/commands/edit.py

@@ -25,13 +25,12 @@ class Command(ScrapyCommand):
         if len(args) != 1:
             raise UsageError()
 
-        crawler = self.crawler_process.create_crawler()
-        editor = crawler.settings['EDITOR']
+        editor = self.settings['EDITOR']
         try:
-            spider = crawler.spiders.create(args[0])
+            spidercls = self.crawler_process.spiders.load(args[0])
         except KeyError:
             return self._err("Spider not found: %s" % args[0])
 
-        sfile = sys.modules[spider.__module__].__file__
+        sfile = sys.modules[spidercls.__module__].__file__
         sfile = sfile.replace('.pyc', '.py')
         self.exitcode = os.system('%s "%s"' % (editor, sfile))

scrapy/commands/genspider.py

@@ -65,15 +65,14 @@ class Command(ScrapyCommand):
             return
 
         try:
-            crawler = self.crawler_process.create_crawler()
-            spider = crawler.spiders.create(name)
+            spidercls = self.crawler_process.spiders.load(name)
         except KeyError:
             pass
         else:
             # if spider already exists and not --force then halt
             if not opts.force:
                 print("Spider %r already exists in module:" % name)
-                print("  %s" % spider.__module__)
+                print("  %s" % spidercls.__module__)
                 return
         template_file = self._find_template(opts.template)
         if template_file:

scrapy/commands/list.py

@@ -10,6 +10,5 @@ class Command(ScrapyCommand):
         return "List available spiders"
 
     def run(self, args, opts):
-        crawler = self.crawler_process.create_crawler()
-        for s in sorted(crawler.spiders.list()):
+        for s in sorted(self.crawler_process.spiders.list()):
             print(s)

scrapy/commands/runspider.py

@@ -83,8 +83,7 @@ class Command(ScrapyCommand):
         spclasses = list(iter_spider_classes(module))
         if not spclasses:
             raise UsageError("No spider found in file: %s\n" % filename)
-        spider = spclasses.pop()(**opts.spargs)
+        spidercls = spclasses.pop()
 
-        crawler = self.crawler_process.create_crawler()
-        crawler.crawl(spider)
+        self.crawler_process.crawl(spidercls, **opts.spargs)
         self.crawler_process.start()

scrapy/contracts/__init__.py

@@ -15,6 +15,15 @@ class ContractsManager(object):
         for contract in contracts:
             self.contracts[contract.name] = contract
 
+    def tested_methods_from_spidercls(self, spidercls):
+        methods = []
+        for key, value in vars(spidercls).items():
+            if (callable(value) and value.__doc__ and
+                    re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
+                methods.append(key)
+
+        return methods
+
     def extract_contracts(self, method):
         contracts = []
         for line in method.__doc__.split('\n'):
@@ -28,6 +37,14 @@ class ContractsManager(object):
 
         return contracts
 
+    def from_spider(self, spider, results):
+        requests = []
+        for method in self.tested_methods_from_spidercls(type(spider)):
+            bound_method = spider.__getattribute__(method)
+            requests.append(self.from_method(bound_method, results))
+
+        return requests
+
     def from_method(self, method, results):
         contracts = self.extract_contracts(method)
         if contracts: