RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

more simplifications to scrapy engine: removed addtasks method

This commit is contained in:
Pablo Hoffman 2009-08-31 12:04:01 -03:00
parent 1dd996f0b6
commit e8e760c974
4 changed files with 44 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scrapy/conf/default_settings.py
View File

@ -119,7 +119,6 @@ MEMUSAGE_REPORT = False
MEMUSAGE_WARNING_MB = 0
MYSQL_CONNECTION_SETTINGS = {}
MYSQL_CONNECTION_PING_PERIOD = 0
NEWSPIDER_MODULE = ''

31
scrapy/contrib/memusage.py
View File

@ -7,8 +7,9 @@ See documentation in docs/topics/extensions.rst
import os
import socket
from scrapy.xlib.pydispatch import dispatcher
from twisted.internet import task
from scrapy.xlib.pydispatch import dispatcher
from scrapy.core import signals
from scrapy import log
from scrapy.core.manager import scrapymanager
@ -26,24 +27,15 @@ class MemoryUsage(object):
raise NotConfigured
if not os.path.exists('/proc'):
raise NotConfigured
self.warned = False
scrapyengine.addtask(self.update, 60.0, now=True)
self.notify_mails = settings.getlist('MEMUSAGE_NOTIFY')
self.limit = settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
self.warning = settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
self.report = settings.getbool('MEMUSAGE_REPORT')
if self.limit:
scrapyengine.addtask(self._check_limit, 60.0, now=True)
if self.warning:
scrapyengine.addtask(self._check_warning, 60.0, now=True)
self.mail = MailSender()
dispatcher.connect(self.engine_started, signal=signals.engine_started)
dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
@property
@ -52,6 +44,23 @@ class MemoryUsage(object):
def engine_started(self):
stats.set_value('memusage/startup', self.virtual)
self.tasks = []
tsk = task.LoopingCall(self.update)
self.tasks.append(tsk)
tsk.start(60.0, now=True)
if self.limit:
tsk = task.LoopingCall(self._check_limit)
self.tasks.append(tsk)
tsk.start(60.0, now=True)
if self.warning:
tsk = task.LoopingCall(self._check_warning)
self.tasks.append(tsk)
tsk.start(60.0, now=True)
def engine_stopped(self):
for tsk in self.tasks:
if tsk.running:
tsk.stop()
def update(self):
stats.max_value('memusage/max', self.virtual)

67
scrapy/core/engine.py
View File

@ -7,7 +7,6 @@ For more information see docs/topics/architecture.rst
from datetime import datetime
from twisted.internet import reactor, task
from twisted.internet.error import CannotListenError
from twisted.python.failure import Failure
from scrapy.xlib.pydispatch import dispatcher
@ -30,11 +29,11 @@ class ExecutionEngine(object):
self.configured = False
self.keep_alive = False
self.closing = {} # dict (domain -> reason) of spiders being closed
self.tasks = []
self.running = False
self.paused = False
self.control_reactor = True
self._next_request_pending = set()
self._mainloop_task = task.LoopingCall(self._mainloop)
def configure(self):
"""
@ -46,52 +45,34 @@ class ExecutionEngine(object):
self.scraper = Scraper(self)
self.configured = True
def addtask(self, function, interval, args=None, kwargs=None, now=False):
"""
Adds a looping task. Use this instead of twisted task.LooopingCall to
make sure the reactor is left in a clean state after the engine is
stopped.
"""
if not args:
args = []
if not kwargs:
kwargs = {}
tsk = task.LoopingCall(function, *args, **kwargs)
self.tasks.append((tsk, interval, now))
if self.running:
tsk.start(interval, now)
return tsk
def start(self, control_reactor=True):
"""Start the execution engine"""
if not self.running:
self.control_reactor = control_reactor
reactor.callLater(0, self._mainloop)
self.start_time = datetime.utcnow()
send_catch_log(signal=signals.engine_started, sender=self.__class__)
self.addtask(self._mainloop, 5.0)
for tsk, interval, now in self.tasks:
tsk.start(interval, now)
self.running = True
if control_reactor:
reactor.run() # blocking call
if self.running:
return
self.control_reactor = control_reactor
self.start_time = datetime.utcnow()
send_catch_log(signal=signals.engine_started, sender=self.__class__)
self._mainloop_task.start(5.0, now=True)
reactor.callWhenRunning(self._mainloop)
self.running = True
if control_reactor:
reactor.run() # blocking call
def stop(self):
"""Stop the execution engine"""
if self.running:
self.running = False
for domain in self.open_domains:
spider = spiders.fromdomain(domain)
send_catch_log(signal=signals.domain_closed, sender=self.__class__, \
domain=domain, spider=spider, reason='shutdown')
stats.close_domain(domain, reason='shutdown')
for tsk, _, _ in self.tasks: # stop looping calls
if tsk.running:
tsk.stop()
self.tasks = []
if self.control_reactor and reactor.running:
reactor.stop()
send_catch_log(signal=signals.engine_stopped, sender=self.__class__)
if not self.running:
return
self.running = False
for domain in self.open_domains:
spider = spiders.fromdomain(domain)
send_catch_log(signal=signals.domain_closed, sender=self.__class__, \
domain=domain, spider=spider, reason='shutdown')
stats.close_domain(domain, reason='shutdown')
if self._mainloop_task.running:
self._mainloop_task.stop()
if self.control_reactor and reactor.running:
reactor.stop()
send_catch_log(signal=signals.engine_stopped, sender=self.__class__)
def pause(self):
"""Pause the execution engine"""

10
scrapy/utils/mysql.py
View File

@ -4,7 +4,6 @@ import re
import MySQLdb
from scrapy.core.engine import scrapyengine
from scrapy.conf import settings
from scrapy import log
@ -37,13 +36,4 @@ def mysql_connect(db_uri_or_dict, **kwargs):
d['host'], d['user']), level=log.DEBUG)
conn = MySQLdb.connect(**d)
# connection keep-alive
def conn_ping():
log.msg("Pinging connection to %s/%s" % (d['host'], d['db']), \
level=log.DEBUG)
conn.ping()
ping_period = settings.getint("MYSQL_CONNECTION_PING_PERIOD")
if ping_period:
scrapyengine.addtask(conn_ping, ping_period)
return conn