From f9b18f9a5920d64c1f0ab8eb005186d16e2341bf Mon Sep 17 00:00:00 2001 From: Alex Cepoi Date: Mon, 19 Aug 2013 23:40:23 +0200 Subject: [PATCH] port all scrapy commands to new CrawlerProcess --- scrapy/command.py | 2 +- scrapy/commands/bench.py | 5 +++-- scrapy/commands/crawl.py | 8 +++++--- scrapy/commands/edit.py | 6 ++++-- scrapy/commands/fetch.py | 10 +++++----- scrapy/commands/genspider.py | 3 ++- scrapy/commands/list.py | 3 ++- scrapy/commands/parse.py | 11 ++++++----- scrapy/commands/runspider.py | 5 +++-- scrapy/commands/settings.py | 3 ++- scrapy/commands/shell.py | 14 ++++++++------ scrapy/crawler.py | 8 ++++++-- 12 files changed, 47 insertions(+), 31 deletions(-) diff --git a/scrapy/command.py b/scrapy/command.py index b7c16b890..039f53442 100644 --- a/scrapy/command.py +++ b/scrapy/command.py @@ -34,7 +34,7 @@ class ScrapyCommand(object): ScrapyDeprecationWarning) if not hasattr(self, '_crawler'): - crawler = self.crawler_process.create_crawler('default') + crawler = self.crawler_process.create_crawler() old_start = crawler.start self.crawler_process.started = False diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 93b3a0552..42b72f6e4 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -16,5 +16,6 @@ class Command(ScrapyCommand): def run(self, args, opts): with MockServer(): spider = FollowAllSpider(total=100000) - self.crawler.crawl(spider) - self.crawler.start() + crawler = self.crawler_process.create_crawler() + crawler.crawl(spider) + self.crawler_process.start() diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index e1167cbc0..9f6bc94e4 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -43,6 +43,8 @@ class Command(ScrapyCommand): elif len(args) > 1: raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") spname = args[0] - spider = self.crawler.spiders.create(spname, **opts.spargs) - self.crawler.crawl(spider) - self.crawler.start() + + crawler = self.crawler_process.create_crawler() + spider = crawler.spiders.create(spname, **opts.spargs) + crawler.crawl(spider) + self.crawler_process.start() diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 90f5e6d4c..b542217e1 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -24,9 +24,11 @@ class Command(ScrapyCommand): def run(self, args, opts): if len(args) != 1: raise UsageError() - editor = self.crawler.settings['EDITOR'] + + crawler = self.crawler_process.create_crawler() + editor = crawler.settings['EDITOR'] try: - spider = self.crawler.spiders.create(args[0]) + spider = crawler.spiders.create(args[0]) except KeyError: return self._err("Spider not found: %s" % args[0]) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index cc4d00f94..c352bf32e 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -47,12 +47,12 @@ class Command(ScrapyCommand): request = Request(args[0], callback=cb, dont_filter=True) request.meta['handle_httpstatus_all'] = True + crawler = self.crawler_process.create_crawler() spider = None if opts.spider: - spider = self.crawler.spiders.create(opts.spider) + spider = crawler.spiders.create(opts.spider) else: - spider = create_spider_for_request(self.crawler.spiders, request, \ + spider = create_spider_for_request(crawler.spiders, request, \ default_spider=BaseSpider('default')) - self.crawler.crawl(spider, [request]) - self.crawler.start() - + crawler.crawl(spider, [request]) + self.crawler_process.start() diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 79deb3ffb..ea4ca003d 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -62,7 +62,8 @@ class Command(ScrapyCommand): return try: - spider = self.crawler.spiders.create(name) + crawler = self.crawler_process.create_crawler() + spider = crawler.spiders.create(name) except KeyError: pass else: diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 906a3f801..d9f7dc175 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -9,5 +9,6 @@ class Command(ScrapyCommand): return "List available spiders" def run(self, args, opts): - for s in self.crawler.spiders.list(): + crawler = self.crawler_process.create_crawler() + for s in crawler.spiders.list(): print s diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ce10eb0b6..7c158cc50 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -125,12 +125,12 @@ class Command(ScrapyCommand): def set_spider(self, url, opts): if opts.spider: try: - self.spider = self.crawler.spiders.create(opts.spider, **opts.spargs) + self.spider = self.pcrawler.spiders.create(opts.spider, **opts.spargs) except KeyError: log.msg(format='Unable to find spider: %(spider)s', level=log.ERROR, spider=opts.spider) else: - self.spider = create_spider_for_request(self.crawler.spiders, Request(url), **opts.spargs) + self.spider = create_spider_for_request(self.pcrawler.spiders, Request(url), **opts.spargs) if not self.spider: log.msg(format='Unable to find spider for: %(url)s', level=log.ERROR, url=url) @@ -139,8 +139,8 @@ class Command(ScrapyCommand): request = Request(url, opts.callback) request = self.prepare_request(request, opts) - self.crawler.crawl(self.spider, [request]) - self.crawler.start() + self.pcrawler.crawl(self.spider, [request]) + self.crawler_process.start() if not self.first_response: log.msg(format='No response downloaded for: %(request)s', @@ -174,7 +174,7 @@ class Command(ScrapyCommand): items, requests = self.run_callback(response, cb) if opts.pipelines: - itemproc = self.crawler.engine.scraper.itemproc + itemproc = self.pcrawler.engine.scraper.itemproc for item in items: itemproc.process_item(item, self.spider) self.add_items(depth, items) @@ -207,6 +207,7 @@ class Command(ScrapyCommand): url = args[0] # prepare spider + self.pcrawler = self.crawler_process.create_crawler() self.set_spider(url, opts) if self.spider and opts.depth > 0: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index ff528fdf3..37e3400dd 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -74,5 +74,6 @@ class Command(ScrapyCommand): raise UsageError("No spider found in file: %s\n" % filename) spider = spclasses.pop()(**opts.spargs) - self.crawler.crawl(spider) - self.crawler.start() + crawler = self.crawler_process.create_crawler() + crawler.crawl(spider) + self.crawler_process.start() diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index dab5d2709..d71546864 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -25,7 +25,8 @@ class Command(ScrapyCommand): help="print setting value, intepreted as an float") def run(self, args, opts): - settings = self.crawler.settings + crawler = self.crawler_process.create_crawler() + settings = crawler.settings if opts.get: print settings.get(opts.get) elif opts.getbool: diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index fe45e7dba..48363b225 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -8,7 +8,7 @@ from threading import Thread from scrapy.command import ScrapyCommand from scrapy.shell import Shell -from scrapy import log + class Command(ScrapyCommand): @@ -38,15 +38,17 @@ class Command(ScrapyCommand): pass def run(self, args, opts): + crawler = self.crawler_process.create_crawler() + url = args[0] if args else None - spider = None - if opts.spider: - spider = self.crawler.spiders.create(opts.spider) - shell = Shell(self.crawler, update_vars=self.update_vars, code=opts.code) + spider = crawler.spiders.create(opts.spider) if opts.spider else None + + shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) self._start_crawler_thread() shell.start(url=url, spider=spider) def _start_crawler_thread(self): - t = Thread(target=self.crawler.start) + self.crawler_process.print_headers() + t = Thread(target=self.crawler_process.start, kwargs={'headers': False}) t.daemon = True t.start() diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 9b6f7bc25..aca3edd6a 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -126,7 +126,7 @@ class CrawlerProcess(ProcessMixin): self.crawlers = {} self.stopping = False - def create_crawler(self, name): + def create_crawler(self, name=None): if name not in self.crawlers: crawler = Crawler(self.settings) crawler.configure() @@ -155,8 +155,12 @@ class CrawlerProcess(ProcessMixin): else: self._stop_reactor() - def start(self): + def print_headers(self): log.scrapy_info(self.settings) + + def start(self, headers=True): + if headers: + self.print_headers() return super(CrawlerProcess, self).start() @defer.inlineCallbacks