RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

port all scrapy commands to new CrawlerProcess

This commit is contained in:
Alex Cepoi 2013-08-19 23:40:23 +02:00 committed by Daniel Graña
parent a5ffdf9bcb
commit f9b18f9a59
12 changed files with 47 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/command.py
View File

@ -34,7 +34,7 @@ class ScrapyCommand(object):
ScrapyDeprecationWarning)
if not hasattr(self, '_crawler'):
crawler = self.crawler_process.create_crawler('default')
crawler = self.crawler_process.create_crawler()
old_start = crawler.start
self.crawler_process.started = False

5
scrapy/commands/bench.py
View File

@ -16,5 +16,6 @@ class Command(ScrapyCommand):
def run(self, args, opts):
with MockServer():
spider = FollowAllSpider(total=100000)
self.crawler.crawl(spider)
self.crawler.start()
crawler = self.crawler_process.create_crawler()
crawler.crawl(spider)
self.crawler_process.start()

8
scrapy/commands/crawl.py
View File

@ -43,6 +43,8 @@ class Command(ScrapyCommand):
elif len(args) > 1:
raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
spname = args[0]
spider = self.crawler.spiders.create(spname, **opts.spargs)
self.crawler.crawl(spider)
self.crawler.start()
crawler = self.crawler_process.create_crawler()
spider = crawler.spiders.create(spname, **opts.spargs)
crawler.crawl(spider)
self.crawler_process.start()

6
scrapy/commands/edit.py
View File

@ -24,9 +24,11 @@ class Command(ScrapyCommand):
def run(self, args, opts):
if len(args) != 1:
raise UsageError()
editor = self.crawler.settings['EDITOR']
crawler = self.crawler_process.create_crawler()
editor = crawler.settings['EDITOR']
try:
spider = self.crawler.spiders.create(args[0])
spider = crawler.spiders.create(args[0])
except KeyError:
return self._err("Spider not found: %s" % args[0])

10
scrapy/commands/fetch.py
View File

@ -47,12 +47,12 @@ class Command(ScrapyCommand):
request = Request(args[0], callback=cb, dont_filter=True)
request.meta['handle_httpstatus_all'] = True
crawler = self.crawler_process.create_crawler()
spider = None
if opts.spider:
spider = self.crawler.spiders.create(opts.spider)
spider = crawler.spiders.create(opts.spider)
else:
spider = create_spider_for_request(self.crawler.spiders, request, \
spider = create_spider_for_request(crawler.spiders, request, \
default_spider=BaseSpider('default'))
self.crawler.crawl(spider, [request])
self.crawler.start()
crawler.crawl(spider, [request])
self.crawler_process.start()

3
scrapy/commands/genspider.py
View File

@ -62,7 +62,8 @@ class Command(ScrapyCommand):
return
try:
spider = self.crawler.spiders.create(name)
crawler = self.crawler_process.create_crawler()
spider = crawler.spiders.create(name)
except KeyError:
pass
else:

3
scrapy/commands/list.py
View File

@ -9,5 +9,6 @@ class Command(ScrapyCommand):
return "List available spiders"
def run(self, args, opts):
for s in self.crawler.spiders.list():
crawler = self.crawler_process.create_crawler()
for s in crawler.spiders.list():
print s

11
scrapy/commands/parse.py
View File

@ -125,12 +125,12 @@ class Command(ScrapyCommand):
def set_spider(self, url, opts):
if opts.spider:
try:
self.spider = self.crawler.spiders.create(opts.spider, **opts.spargs)
self.spider = self.pcrawler.spiders.create(opts.spider, **opts.spargs)
except KeyError:
log.msg(format='Unable to find spider: %(spider)s',
level=log.ERROR, spider=opts.spider)
else:
self.spider = create_spider_for_request(self.crawler.spiders, Request(url), **opts.spargs)
self.spider = create_spider_for_request(self.pcrawler.spiders, Request(url), **opts.spargs)
if not self.spider:
log.msg(format='Unable to find spider for: %(url)s',
level=log.ERROR, url=url)
@ -139,8 +139,8 @@ class Command(ScrapyCommand):
request = Request(url, opts.callback)
request = self.prepare_request(request, opts)
self.crawler.crawl(self.spider, [request])
self.crawler.start()
self.pcrawler.crawl(self.spider, [request])
self.crawler_process.start()
if not self.first_response:
log.msg(format='No response downloaded for: %(request)s',
@ -174,7 +174,7 @@ class Command(ScrapyCommand):
items, requests = self.run_callback(response, cb)
if opts.pipelines:
itemproc = self.crawler.engine.scraper.itemproc
itemproc = self.pcrawler.engine.scraper.itemproc
for item in items:
itemproc.process_item(item, self.spider)
self.add_items(depth, items)
@ -207,6 +207,7 @@ class Command(ScrapyCommand):
url = args[0]
# prepare spider
self.pcrawler = self.crawler_process.create_crawler()
self.set_spider(url, opts)
if self.spider and opts.depth > 0:

5
scrapy/commands/runspider.py
View File

@ -74,5 +74,6 @@ class Command(ScrapyCommand):
raise UsageError("No spider found in file: %s\n" % filename)
spider = spclasses.pop()(**opts.spargs)
self.crawler.crawl(spider)
self.crawler.start()
crawler = self.crawler_process.create_crawler()
crawler.crawl(spider)
self.crawler_process.start()

3
scrapy/commands/settings.py
View File

@ -25,7 +25,8 @@ class Command(ScrapyCommand):
help="print setting value, intepreted as an float")
def run(self, args, opts):
settings = self.crawler.settings
crawler = self.crawler_process.create_crawler()
settings = crawler.settings
if opts.get:
print settings.get(opts.get)
elif opts.getbool:

14
scrapy/commands/shell.py
View File

@ -8,7 +8,7 @@ from threading import Thread
from scrapy.command import ScrapyCommand
from scrapy.shell import Shell
from scrapy import log
class Command(ScrapyCommand):
@ -38,15 +38,17 @@ class Command(ScrapyCommand):
pass
def run(self, args, opts):
crawler = self.crawler_process.create_crawler()
url = args[0] if args else None
spider = None
if opts.spider:
spider = self.crawler.spiders.create(opts.spider)
shell = Shell(self.crawler, update_vars=self.update_vars, code=opts.code)
spider = crawler.spiders.create(opts.spider) if opts.spider else None
shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
self._start_crawler_thread()
shell.start(url=url, spider=spider)
def _start_crawler_thread(self):
t = Thread(target=self.crawler.start)
self.crawler_process.print_headers()
t = Thread(target=self.crawler_process.start, kwargs={'headers': False})
t.daemon = True
t.start()

8
scrapy/crawler.py
View File

@ -126,7 +126,7 @@ class CrawlerProcess(ProcessMixin):
self.crawlers = {}
self.stopping = False
def create_crawler(self, name):
def create_crawler(self, name=None):
if name not in self.crawlers:
crawler = Crawler(self.settings)
crawler.configure()
@ -155,8 +155,12 @@ class CrawlerProcess(ProcessMixin):
else:
self._stop_reactor()
def start(self):
def print_headers(self):
log.scrapy_info(self.settings)
def start(self, headers=True):
if headers:
self.print_headers()
return super(CrawlerProcess, self).start()
@defer.inlineCallbacks