diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 18934f976..395597546 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -22,9 +22,7 @@ class Command(ScrapyCommand): def run(self, args, opts): with _BenchServer(): - spider = _BenchSpider(total=100000) - crawler = self.crawler_process.create_crawler() - crawler.crawl(spider) + self.crawler_process.crawl(_BenchSpider, total=100000) self.crawler_process.start() diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 40ff9014b..014b00eeb 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -69,20 +69,18 @@ class Command(ScrapyCommand): # contract requests contract_reqs = defaultdict(list) - spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) - spiders = spman_cls.from_settings(self.settings) + spiders = self.crawler_process.spiders - for spider in args or spiders.list(): - spider = spiders.create(spider) - requests = self.get_requests(spider, conman, result) - contract_reqs[spider.name] = [] + for spidername in args or spiders.list(): + spidercls = spiders.load(spidername) + spidercls.start_requests = lambda s: conman.from_spider(s, result) + tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: - for req in requests: - contract_reqs[spider.name].append(req.callback.__name__) - elif requests: - crawler = self.crawler_process.create_crawler(spider.name) - crawler.crawl(spider, requests) + for method in tested_methods: + contract_reqs[spidercls.name].append(method) + elif tested_methods: + self.crawler_process.crawl(spidercls) # start checks if opts.list: @@ -101,15 +99,3 @@ class Command(ScrapyCommand): result.printSummary(start, stop) self.exitcode = int(not result.wasSuccessful()) - def get_requests(self, spider, conman, result): - requests = [] - - for key, value in vars(type(spider)).items(): - if callable(value) and value.__doc__: - bound_method = value.__get__(spider, type(spider)) - request = conman.from_method(bound_method, result) - - if request: - requests.append(request) - - return requests diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 0976de51b..b7fea7b80 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -54,7 +54,5 @@ class Command(ScrapyCommand): raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") spname = args[0] - crawler = self.crawler_process.create_crawler() - spider = crawler.spiders.create(spname, **opts.spargs) - crawler.crawl(spider) + self.crawler_process.crawl(spname, **opts.spargs) self.crawler_process.start() diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index b542217e1..e20e7c2e5 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -25,13 +25,12 @@ class Command(ScrapyCommand): if len(args) != 1: raise UsageError() - crawler = self.crawler_process.create_crawler() - editor = crawler.settings['EDITOR'] + editor = self.settings['EDITOR'] try: - spider = crawler.spiders.create(args[0]) + spidercls = self.crawler_process.spiders.load(args[0]) except KeyError: return self._err("Spider not found: %s" % args[0]) - sfile = sys.modules[spider.__module__].__file__ + sfile = sys.modules[spidercls.__module__].__file__ sfile = sfile.replace('.pyc', '.py') self.exitcode = os.system('%s "%s"' % (editor, sfile)) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 3e2e24b21..52c5d9f94 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -65,15 +65,14 @@ class Command(ScrapyCommand): return try: - crawler = self.crawler_process.create_crawler() - spider = crawler.spiders.create(name) + spidercls = self.crawler_process.spiders.load(name) except KeyError: pass else: # if spider already exists and not --force then halt if not opts.force: print("Spider %r already exists in module:" % name) - print(" %s" % spider.__module__) + print(" %s" % spidercls.__module__) return template_file = self._find_template(opts.template) if template_file: diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 0ea9c2313..2d55d59bd 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -10,6 +10,5 @@ class Command(ScrapyCommand): return "List available spiders" def run(self, args, opts): - crawler = self.crawler_process.create_crawler() - for s in sorted(crawler.spiders.list()): + for s in sorted(self.crawler_process.spiders.list()): print(s) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index b1d501842..b67838619 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -83,8 +83,7 @@ class Command(ScrapyCommand): spclasses = list(iter_spider_classes(module)) if not spclasses: raise UsageError("No spider found in file: %s\n" % filename) - spider = spclasses.pop()(**opts.spargs) + spidercls = spclasses.pop() - crawler = self.crawler_process.create_crawler() - crawler.crawl(spider) + self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.start() diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 03e6e4e0c..5eaee3d11 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -15,6 +15,15 @@ class ContractsManager(object): for contract in contracts: self.contracts[contract.name] = contract + def tested_methods_from_spidercls(self, spidercls): + methods = [] + for key, value in vars(spidercls).items(): + if (callable(value) and value.__doc__ and + re.search(r'^\s*@', value.__doc__, re.MULTILINE)): + methods.append(key) + + return methods + def extract_contracts(self, method): contracts = [] for line in method.__doc__.split('\n'): @@ -28,6 +37,14 @@ class ContractsManager(object): return contracts + def from_spider(self, spider, results): + requests = [] + for method in self.tested_methods_from_spidercls(type(spider)): + bound_method = spider.__getattribute__(method) + requests.append(self.from_method(bound_method, results)) + + return requests + def from_method(self, method, results): contracts = self.extract_contracts(method) if contracts: