diff --git a/docs/topics/scrapyd.rst b/docs/topics/scrapyd.rst index 752bb80b6..2327e6d0f 100644 --- a/docs/topics/scrapyd.rst +++ b/docs/topics/scrapyd.rst @@ -189,10 +189,16 @@ logs_dir The directory where the Scrapy processes logs will be stored. -logs_to_keep +items_dir +--------- + +The directory where the Scrapy items will be stored. + +jobs_to_keep ------------ -The number of logs to keep per spider. Defaults to ``5``. +The number of finished jobs to keep per spider. Defaults to ``5``. This +includes logs and items. runner ------ diff --git a/scrapyd/default_scrapyd.conf b/scrapyd/default_scrapyd.conf index 5680002cb..de777e515 100644 --- a/scrapyd/default_scrapyd.conf +++ b/scrapyd/default_scrapyd.conf @@ -1,7 +1,8 @@ [scrapyd] eggs_dir = eggs logs_dir = logs -logs_to_keep = 5 +items_dir = items +jobs_to_keep = 5 dbs_dir = dbs max_proc = 0 max_proc_per_cpu = 4 diff --git a/scrapyd/environ.py b/scrapyd/environ.py index 0f2fc89c8..69a6aad48 100644 --- a/scrapyd/environ.py +++ b/scrapyd/environ.py @@ -11,7 +11,8 @@ class Environment(object): def __init__(self, config, initenv=os.environ): self.dbs_dir = config.get('dbs_dir', 'dbs') self.logs_dir = config.get('logs_dir', 'logs') - self.logs_to_keep = config.getint('logs_to_keep', 5) + self.items_dir = config.get('items_dir', 'items') + self.jobs_to_keep = config.getint('jobs_to_keep', 5) if config.cp.has_section('settings'): self.settings = dict(config.cp.items('settings')) else: @@ -29,15 +30,17 @@ class Environment(object): env['SCRAPY_SETTINGS_MODULE'] = self.settings[project] dbpath = os.path.join(self.dbs_dir, '%s.db' % project) env['SCRAPY_SQLITE_DB'] = dbpath - env['SCRAPY_LOG_FILE'] = self._get_log_file(message) + env['SCRAPY_LOG_FILE'] = self._get_file(message, self.logs_dir, 'log') + env['SCRAPY_FEED_URI'] = self._get_file(message, self.items_dir, 'jl') return env - def _get_log_file(self, message): - logsdir = os.path.join(self.logs_dir, message['_project'], \ + def _get_file(self, message, dir, ext): + logsdir = os.path.join(dir, message['_project'], \ message['_spider']) if not os.path.exists(logsdir): os.makedirs(logsdir) - to_delete = sorted((os.path.join(logsdir, x) for x in os.listdir(logsdir)), key=os.path.getmtime)[:-self.logs_to_keep] + to_delete = sorted((os.path.join(logsdir, x) for x in \ + os.listdir(logsdir)), key=os.path.getmtime)[:-self.jobs_to_keep] for x in to_delete: os.remove(x) - return os.path.join(logsdir, "%s.log" % message['_job']) + return os.path.join(logsdir, "%s.%s" % (message['_job'], ext)) diff --git a/scrapyd/launcher.py b/scrapyd/launcher.py index 270209ebc..f1b219bea 100644 --- a/scrapyd/launcher.py +++ b/scrapyd/launcher.py @@ -68,6 +68,7 @@ class ScrapyProcessProtocol(protocol.ProcessProtocol): self.end_time = None self.env = env self.logfile = env['SCRAPY_LOG_FILE'] + self.itemsfile = env['SCRAPY_FEED_URI'] self.deferred = defer.Deferred() def outReceived(self, data): @@ -88,6 +89,6 @@ class ScrapyProcessProtocol(protocol.ProcessProtocol): self.deferred.callback(self) def log(self, msg): - msg += "project=%r spider=%r job=%r pid=%r log=%r" % (self.project, \ - self.spider, self.job, self.pid, self.logfile) + msg += "project=%r spider=%r job=%r pid=%r log=%r items=%r" % (self.project, \ + self.spider, self.job, self.pid, self.logfile, self.itemsfile) log.msg(msg, system="Launcher") diff --git a/scrapyd/script.py b/scrapyd/script.py index ce0b3af92..5c2711a33 100644 --- a/scrapyd/script.py +++ b/scrapyd/script.py @@ -18,9 +18,10 @@ def _get_config(): conf = { 'eggs_dir': os.path.join(datadir, 'eggs'), 'logs_dir': os.path.join(datadir, 'logs'), + 'items_dir': os.path.join(datadir, 'items'), 'dbs_dir': os.path.join(datadir, 'dbs'), } - for k in ['eggs_dir', 'logs_dir', 'dbs_dir']: # create dirs + for k in ['eggs_dir', 'logs_dir', 'items_dir', 'dbs_dir']: # create dirs d = conf[k] if not os.path.exists(d): os.makedirs(d) @@ -28,6 +29,7 @@ def _get_config(): [scrapyd] eggs_dir = %(eggs_dir)s logs_dir = %(logs_dir)s +items_dir = %(items_dir)s dbs_dir = %(dbs_dir)s """ % conf return Config(extra_sources=[StringIO(scrapyd_conf)]) diff --git a/scrapyd/tests/test_environ.py b/scrapyd/tests/test_environ.py index 220145fd2..a90b8c9c3 100644 --- a/scrapyd/tests/test_environ.py +++ b/scrapyd/tests/test_environ.py @@ -31,4 +31,5 @@ class EnvironmentTest(unittest.TestCase): self.assertEqual(env['SCRAPY_JOB'], 'ID') self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db')) self.assert_(env['SCRAPY_LOG_FILE'].endswith(os.path.join('mybot', 'myspider', 'ID.log'))) + self.assert_(env['SCRAPY_FEED_URI'].endswith(os.path.join('mybot', 'myspider', 'ID.jl'))) self.failIf('SCRAPY_SETTINGS_MODULE' in env) diff --git a/scrapyd/website.py b/scrapyd/website.py index 3513151e0..1b9fd9706 100644 --- a/scrapyd/website.py +++ b/scrapyd/website.py @@ -13,6 +13,7 @@ class Root(resource.Resource): self.debug = config.getboolean('debug', False) self.runner = config.get('runner') logsdir = config.get('logs_dir') + itemsdir = config.get('items_dir') self.app = app self.putChild('', Home(self)) self.putChild('schedule.json', webservice.Schedule(self)) @@ -25,6 +26,7 @@ class Root(resource.Resource): self.putChild('delversion.json', webservice.DeleteVersion(self)) self.putChild('listjobs.json', webservice.ListJobs(self)) self.putChild('logs', static.File(logsdir, 'text/plain')) + self.putChild('items', static.File(itemsdir, 'text/plain')) self.putChild('jobs', Jobs(self)) self.update_projects() @@ -68,6 +70,7 @@ class Home(resource.Resource):

Available projects: %(projects)s

@@ -98,8 +101,8 @@ class Jobs(resource.Resource): s += "

Jobs

" s += "

Go back

" s += "" - s += "" - s += "" + s += "" + s += "" for project, queue in self.root.poller.queues.items(): for m in queue.list(): s += "" @@ -107,15 +110,16 @@ class Jobs(resource.Resource): s += "" % str(m['name']) s += "" % str(m['_job']) s += "" - s += "" + s += "" for p in self.root.launcher.processes.values(): s += "" for a in ['project', 'spider', 'job', 'pid']: s += "" % getattr(p, a) s += "" % (datetime.now() - p.start_time) s += "" % (p.project, p.spider, p.job) + s += "" % (p.project, p.spider, p.job) s += "" - s += "" + s += "" for p in self.root.launcher.finished: s += "" for a in ['project', 'spider', 'job']:
ProjectSpiderJobPIDRuntimeLog
Pending
ProjectSpiderJobPIDRuntimeLogItems
Pending
%s%s
Running
Running
%s%sLogItems
Finished
Finished