RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

scrapyd: changed keys used in poller message to _project, _spider, _job, and added link to log file in web ui

This commit is contained in:
Pablo Hoffman 2010-11-30 13:03:20 -02:00
parent df54ed0041
commit 5c4f562ec4
10 changed files with 32 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapyd/environ.py
View File

@ -19,7 +19,7 @@ class Environment(object):
self.initenv = initenv
def get_environment(self, message, slot, eggpath):
project = message['project']
project = message['_project']
env = self.initenv.copy()
env['SCRAPY_PROJECT'] = project
if eggpath:
@ -32,11 +32,11 @@ class Environment(object):
return env
def _get_log_file(self, message):
logsdir = os.path.join(self.logs_dir, message['project'], \
message['spider'])
logsdir = os.path.join(self.logs_dir, message['_project'], \
message['_spider'])
if not os.path.exists(logsdir):
os.makedirs(logsdir)
to_delete = sorted(os.listdir(logsdir), reverse=True)[:-self.logs_to_keep]
for x in to_delete:
os.remove(os.path.join(logsdir, x))
return os.path.join(logsdir, "%s.log" % message['_id'])
return os.path.join(logsdir, "%s.log" % message['_job'])

12
scrapyd/interfaces.py
View File

@ -35,9 +35,11 @@ class IPoller(Interface):
project that needs to run, or already fired if there was a project
waiting to run already.
The message is a dict containing (at least) the name of the project to
be run in the 'project' key. This message will be passed later to
IEnvironment.get_environment().
The message is a dict containing (at least):
* the name of the project to be run in the '_project' key
* the name of the spider to be run in the '_spider' key
* a unique identifier for this run in the `_job` key
This message will be passed later to IEnvironment.get_environment().
"""
def update_projects():
@ -65,9 +67,7 @@ class IEnvironment(Interface):
def get_environment(message, slot, eggpath):
"""Return the environment variables to use for running the process.
`message` is the message received from the IPoller.next() augmented to
contain (at least) the following keys: project, spider, _id (where
_id is a unique identifier for this run)
`message` is the message received from the IPoller.next() method
`slot` is the Launcher slot where the process will be running.
`eggpath` is the path to an eggfile that contains the project code. The
`eggpath` may be `None` if no egg was found for the project, in

10
scrapyd/launcher.py
View File

@ -48,13 +48,15 @@ class Launcher(Service):
def _spawn_process(self, message, slot):
msg = stringify_dict(message, keys_only=False)
project = msg['project']
project = msg['_project']
eggpath = self._get_eggpath(project)
args = [sys.executable, '-m', self.egg_runner, 'crawl']
args += get_crawl_args(msg)
e = self.app.getComponent(IEnvironment)
env = e.get_environment(msg, slot, eggpath)
pp = ScrapyProcessProtocol(eggpath, slot, project, msg['spider'], msg['_id'])
env = stringify_dict(env, keys_only=False)
pp = ScrapyProcessProtocol(eggpath, slot, project, msg['_spider'], \
msg['_job'], env)
pp.deferred.addBoth(self._process_finished, eggpath, slot)
reactor.spawnProcess(pp, sys.executable, args=args, env=env)
self.processes[slot] = pp
@ -68,7 +70,7 @@ class Launcher(Service):
class ScrapyProcessProtocol(protocol.ProcessProtocol):
def __init__(self, eggfile, slot, project, spider, job):
def __init__(self, eggfile, slot, project, spider, job, env):
self.eggfile = eggfile
self.slot = slot
self.pid = None
@ -76,6 +78,8 @@ class ScrapyProcessProtocol(protocol.ProcessProtocol):
self.spider = spider
self.job = job
self.start_time = datetime.now()
self.env = env
self.logfile = env['SCRAPY_LOG_FILE']
self.deferred = defer.Deferred()
def outReceived(self, data):

4
scrapyd/poller.py
View File

@ -29,6 +29,6 @@ class QueuePoller(object):
def _message(self, queue_msg, project):
d = queue_msg.copy()
d['project'] = project
d['spider'] = d.pop('name')
d['_project'] = project
d['_spider'] = d.pop('name')
return d

4
scrapyd/tests/test_environ.py
View File

@ -22,7 +22,7 @@ class EggStorageTest(unittest.TestCase):
verifyObject(IEnvironment, self.environ)
def test_get_environment_with_eggfile(self):
msg = {'project': 'mybot', 'spider': 'myspider', '_id': 'ID'}
msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
slot = 3
env = self.environ.get_environment(msg, slot, '/path/to/file.egg')
self.assertEqual(env['SCRAPY_PROJECT'], 'mybot')
@ -32,7 +32,7 @@ class EggStorageTest(unittest.TestCase):
self.failIf('SCRAPY_SETTINGS_MODULE' in env)
def test_get_environment_without_eggfile(self):
msg = {'project': 'newbot', 'spider': 'myspider', '_id': 'ID'}
msg = {'_project': 'newbot', '_spider': 'myspider', '_job': 'ID'}
slot = 3
env = self.environ.get_environment(msg, slot, None)
self.assertEqual(env['SCRAPY_PROJECT'], 'newbot')

4
scrapyd/tests/test_poller.py
View File

@ -37,5 +37,5 @@ class QueuePollerTest(unittest.TestCase):
self.poller.poll()
self.queues['mybot1'].pop()
self.poller.poll()
self.failUnlessEqual(d1.result, {'project': 'mybot1', 'spider': 'spider1'})
self.failUnlessEqual(d2.result, {'project': 'mybot2', 'spider': 'spider2'})
self.failUnlessEqual(d1.result, {'_project': 'mybot1', '_spider': 'spider1'})
self.failUnlessEqual(d2.result, {'_project': 'mybot2', '_spider': 'spider2'})

4
scrapyd/tests/test_utils.py
View File

@ -5,9 +5,9 @@ from scrapyd.utils import get_crawl_args
class UtilsTest(unittest.TestCase):
def test_get_crawl_args(self):
msg = {'project': 'lolo', 'spider': 'lala'}
msg = {'_project': 'lolo', '_spider': 'lala'}
self.assertEqual(get_crawl_args(msg), ['lala'])
msg = {'project': 'lolo', 'spider': 'lala', 'arg1': u'val1'}
msg = {'_project': 'lolo', '_spider': 'lala', 'arg1': u'val1'}
cargs = get_crawl_args(msg)
self.assertEqual(cargs, ['lala', '-a', 'arg1=val1'])
assert all(isinstance(x, str) for x in cargs), cargs

4
scrapyd/utils.py
View File

@ -35,8 +35,8 @@ def get_crawl_args(message):
that will be started for this message
"""
msg = message.copy()
args = [unicode_to_str(msg['spider'])]
del msg['project'], msg['spider']
args = [unicode_to_str(msg['_spider'])]
del msg['_project'], msg['_spider']
for k, v in stringify_dict(msg, keys_only=False).items():
args += ['-a']
args += ['%s=%s' % (k, v)]

2
scrapyd/webservice.py
View File

@ -28,7 +28,7 @@ class Schedule(WsResource):
project = args.pop('project')
spider = args.pop('spider')
jobid = uuid.uuid1().hex
args['_id'] = jobid
args['_job'] = jobid
self.root.scheduler.schedule(project, spider, **args)
jobids = {spider: jobid}
return {"status": "ok", "jobs": jobids}

7
scrapyd/website.py
View File

@ -57,8 +57,8 @@ class Home(resource.Resource):
<body>
<h1>Scrapyd</h1>
<ul>
<li><a href="procmon">Process monitor</a></li>
<li><a href="logs/">Logs</li>
<li><a href="/procmon">Process monitor</a></li>
<li><a href="/logs/">Logs</li>
<li><a href="http://doc.scrapy.org/topics/scrapyd.html">Documentation</a></li>
</ul>
</body>
@ -79,13 +79,14 @@ class ProcessMonitor(resource.Resource):
s += "<p><a href='..'>Go back</a></p>"
s += "<table border='1'>"
s += "<tr>"
s += "<th>Project</th><th>Spider</th><th>Job</th><th>PID</th><th>Runtime</th>"
s += "<th>Project</th><th>Spider</th><th>Job</th><th>PID</th><th>Runtime</th><th>Log</th>"
s += "</tr>"
for p in self.root.launcher.processes.values():
s += "<tr>"
for a in ['project', 'spider', 'job', 'pid']:
s += "<td>%s</td>" % getattr(p, a)
s += "<td>%s</td>" % (datetime.now() - p.start_time)
s += "<td><a href='/logs/%s/%s/%s.log'>Log</a></td>" % (p.project, p.spider, p.job)
s += "</tr>"
s += "</table>"
s += "</body>"