Commit af444b6a authored by Marius Isken's avatar Marius Isken
Browse files

wip baraddur / grond

parent c4add9b8
...@@ -31,6 +31,7 @@ subcommand_descriptions = { ...@@ -31,6 +31,7 @@ subcommand_descriptions = {
'forward': 'run forward modelling', 'forward': 'run forward modelling',
'harvest': 'manually run harvesting', 'harvest': 'manually run harvesting',
'plot': 'plot optimization result', 'plot': 'plot optimization result',
'baraddur': 'start Barad-dur plotting, watching this directory',
'export': 'export results', 'export': 'export results',
} }
...@@ -44,6 +45,7 @@ subcommand_usages = { ...@@ -44,6 +45,7 @@ subcommand_usages = {
'forward <configfile> <eventnames> ... [options]'), 'forward <configfile> <eventnames> ... [options]'),
'harvest': 'harvest <rundir> [options]', 'harvest': 'harvest <rundir> [options]',
'plot': 'plot <plotnames> <rundir> [options]', 'plot': 'plot <plotnames> <rundir> [options]',
'baraddur': 'plot-server',
'export': 'export (best|mean|ensemble|stats) <rundirs> ... [options]', 'export': 'export (best|mean|ensemble|stats) <rundirs> ... [options]',
} }
...@@ -65,6 +67,7 @@ Subcommands: ...@@ -65,6 +67,7 @@ Subcommands:
forward %(forward)s forward %(forward)s
harvest %(harvest)s harvest %(harvest)s
plot %(plot)s plot %(plot)s
baraddur %(baraddur)s
export %(export)s export %(export)s
To get further help and a list of available options for any subcommand run: To get further help and a list of available options for any subcommand run:
...@@ -460,6 +463,31 @@ selected by specifying a comma-separated list.''' % ( ...@@ -460,6 +463,31 @@ selected by specifying a comma-separated list.''' % (
die(str(e)) die(str(e))
def command_baraddur(args):
from grond.baraddur import Baraddur, BaraddurConfig
import signal
def setup(parser):
parser.add_option(
'--address', dest='address', default='*',
help='Address listening on, default all \'*\'')
parser.add_option(
'--port', dest='port', type='int', default=8080,
help='Port to listen on, default 8080')
parser, options, args = cl_parse('baraddur', args, setup)
config = BaraddurConfig(
project_dir=op.abspath(op.curdir),
hosts=options.address,
port=options.port)
baraddur = Baraddur(config=config)
signal.signal(signal.SIGINT, baraddur.stop)
baraddur.start()
def command_export(args): def command_export(args):
def setup(parser): def setup(parser):
......
...@@ -13,6 +13,6 @@ setup( ...@@ -13,6 +13,6 @@ setup(
package_dir={'grond': 'src'}, package_dir={'grond': 'src'},
package_data={'grond': [], package_data={'grond': [],
'grond': ['baraddur/templates/*.html', 'grond': ['baraddur/templates/*.html',
'baraddur/css/*']}, 'baraddur/res/*']},
data_files=[('/etc/bash_completion.d', ['extras/grond'])], data_files=[('/etc/bash_completion.d', ['extras/grond'])],
) )
import multiprocessing as _mp import multiprocessing as _mp
import signal as _signal import signal as _signal
from .server import Baraddur # noqa from .server import Baraddur, BaraddurConfig # noqa
class BaraddurProcess(_mp.Process): class BaraddurProcess(_mp.Process):
...@@ -9,9 +9,9 @@ class BaraddurProcess(_mp.Process): ...@@ -9,9 +9,9 @@ class BaraddurProcess(_mp.Process):
self.server = Baraddur(*args, **kwargs) self.server = Baraddur(*args, **kwargs)
self.shutdown_signal = _mp.Queue(1) self.shutdown_signal = _mp.Queue(1)
_mp.Process.__init__(self) _mp.Process.__init__(self)
_signal.signal(_signal.SIGINT, self.server.stop)
def run(self): def run(self):
_signal.signal(_signal.SIGINT, self.server.stop)
self.server.start(signal=self.shutdown_signal) self.server.start(signal=self.shutdown_signal)
def stop(self): def stop(self):
......
function changeRundir(rundir) {
var form = document.createElement("form");
//Move the submit function to another variable
//so that it doesn't get overwritten.
form._submit_function_ = form.submit;
form.setAttribute("method", "post");
form.setAttribute("action", window.location.href);
var rundirField = document.createElement("input");
rundirField.setAttribute("type", "hidden");
rundirField.setAttribute("name", "rundir");
rundirField.setAttribute("value", rundir);
form.appendChild(rundirField);
document.body.appendChild(form);
form._submit_function_(); //Call the renamed function.
}
...@@ -41,8 +41,8 @@ The content `<div>` is where all your content goes. ...@@ -41,8 +41,8 @@ The content `<div>` is where all your content goes.
*/ */
div.footer { div.footer {
margin: 1em 0em; margin: 1em 0em 0em 0em;
padding: 0em 0em; padding: 0em 0em 1em 0em;
line-height: 3em; line-height: 3em;
text-align: center; text-align: center;
color: #999; color: #999;
...@@ -269,3 +269,8 @@ Hides the menu at `48em`, but modify this based on your app's needs. ...@@ -269,3 +269,8 @@ Hides the menu at `48em`, but modify this based on your app's needs.
left: 200px; left: 200px;
} }
} }
.button-selected {
background: rgb(28, 184, 65); /* this is a green */
}
import tornado.ioloop import tornado.ioloop
import grond import grond
import os import os
import glob
import os.path as op import os.path as op
import logging import logging
import numpy as num import numpy as num
import socket import socket
from collections import OrderedDict from collections import OrderedDict
from datetime import date
from pyrocko.guts import Object, Bool, String, Int, List from pyrocko.guts import Object, Bool, String, Int, List
...@@ -47,27 +49,66 @@ class BaraddurBokehHandler(BokehHandler): ...@@ -47,27 +49,66 @@ class BaraddurBokehHandler(BokehHandler):
class GrondBokehModel(object): class GrondBokehModel(object):
def __init__(self, config): def __init__(self, rundir):
self.config = config self.set_rundir(rundir)
self.set_rundir(self.config.rundir)
def set_rundir(self, rundir): def set_rundir(self, rundir):
logger.debug('Loading problem from %s' % rundir) logger.debug('Loading problem from %s' % rundir)
self.rundir = rundir self.rundir = op.abspath(rundir)
self.problem = grond.core.load_problem_info(self.rundir) self.problem = grond.core.load_problem_info(self.rundir)
self.parameters = self.problem.parameters self.parameters = self.problem.parameters
self.nparameters = self.problem.nparameters self.nparameters = self.problem.nparameters
self.ntargets = self.problem.ntargets self.ntargets = self.problem.ntargets
def get_models(self, skip_nmodels=0): def _jp(self, file):
fn = op.join(self.rundir, 'models') return op.join(self.rundir, file)
@property
def start_time(self):
stat = os.stat(self._jp('problem.yaml'))
try:
t = stat.st_birthtime
except AttributeError:
t = stat.st_mtime
return date.fromtimestamp(t).strftime('%Y-%m-%d %H:%M')
@property
def name(self):
return op.split(self.rundir)[-1]
@property
def best_misfit(self):
_, data = self.get_misfits(keys='target_mean')
return data['target_mean'].min()
@property
def niterations(self):
return op.getsize(self._jp('models')) / (self.nparameters * 8)
@staticmethod
def validate_rundir(rundir):
def jp(fn):
return op.join(rundir, fn)
for fn in ['models', 'misfits', 'problem.yaml']:
if op.lexists(jp(fn)):
if op.getsize(jp(fn)) == 0:
return False
else:
return False
return True
def get_models(self, skip_nmodels=0, keys=None):
fn = self._jp('models')
with open(fn, 'r') as f: with open(fn, 'r') as f:
nmodels = os.fstat(f.fileno()).st_size / (self.nparameters * 8) nmodels = os.fstat(f.fileno()).st_size / (self.nparameters * 8)
nmodels -= skip_nmodels nmodels -= skip_nmodels
f.seek(skip_nmodels * self.nparameters * 8) f.seek(skip_nmodels * self.nparameters * 8)
data = num.fromfile( data = num.fromfile(
f, dtype='<f8', count=nmodels * self.nparameters)\ f, dtype='<f8', count=nmodels * self.nparameters)\
.astype(num.float) .astype(num.float32)
nmodels = data.size/self.nparameters - skip_nmodels nmodels = data.size/self.nparameters - skip_nmodels
models = data.reshape((nmodels, self.nparameters)) models = data.reshape((nmodels, self.nparameters))
...@@ -75,11 +116,18 @@ class GrondBokehModel(object): ...@@ -75,11 +116,18 @@ class GrondBokehModel(object):
mods_dict = {} mods_dict = {}
for ip, par in enumerate(self.parameters): for ip, par in enumerate(self.parameters):
mods_dict[par.name] = models[:, ip] mods_dict[par.name] = models[:, ip]
mods_dict['niter'] = num.arange(nmodels, dtype=num.int) + (nmodels+1) mods_dict['niter'] = num.arange(nmodels, dtype=num.int)\
+ skip_nmodels + 1
if keys is not None:
for k in mods_dict.keys():
if k not in keys:
mods_dict.pop(k)
return nmodels, mods_dict return nmodels, mods_dict
def get_misfits(self, skip_nmodels=0): def get_misfits(self, skip_nmodels=0, keys=None):
fn = op.join(self.rundir, 'misfits') fn = self._jp('misfits')
with open(fn, 'r') as f: with open(fn, 'r') as f:
nmodels = os.fstat(f.fileno()).st_size / (self.nparameters * 8) nmodels = os.fstat(f.fileno()).st_size / (self.nparameters * 8)
...@@ -87,24 +135,55 @@ class GrondBokehModel(object): ...@@ -87,24 +135,55 @@ class GrondBokehModel(object):
f.seek(skip_nmodels * self.ntargets * 2 * 8) f.seek(skip_nmodels * self.ntargets * 2 * 8)
data = num.fromfile( data = num.fromfile(
f, dtype='<f8', count=nmodels*self.ntargets*2)\ f, dtype='<f8', count=nmodels*self.ntargets*2)\
.astype(num.float) .astype(num.float32)
data = data.reshape((nmodels, self.ntargets*2)) data = data.reshape((nmodels, self.ntargets*2))
combi = num.empty_like(data) combi = num.empty_like(data)
combi[:, 0::2] = data[:, :self.ntargets] combi[:, 0::2] = data[:, :self.ntargets]
combi[:, 1::2] = data[:, self.ntargets:] combi[:, 1::2] = data[:, self.ntargets:]
assert(data.size/self.nparameters - skip_nmodels == nmodels)
misfits = combi.reshape((nmodels, self.ntargets, 2)) misfits = combi.reshape((nmodels, self.ntargets, 2))
mf_dict = {} mf_dict = {}
for it in xrange(self.ntargets): for it in xrange(self.ntargets):
mf_dict['target_%03d' % it] = misfits[:, it, 0] mf_dict['target_%03d' % it] = misfits[:, it, 0]
mf_dict['target_mean'] = num.mean(misfits[:, :, 0]) mf_dict['target_mean'] = num.mean(misfits, axis=1)[:, 0]
mf_dict['niter'] = num.arange(nmodels, dtype=num.int) + (nmodels+1) mf_dict['niter'] = num.arange(nmodels, dtype=num.int)\
return nmodels, misfits + skip_nmodels + 1
if keys is not None:
for k in mf_dict.keys():
if k not in keys:
mf_dict.pop(k)
return nmodels, mf_dict
class Rundirs(BaraddurRequestHandler):
def get(self):
models = [GrondBokehModel(rundir=rd)
for rd in self.config.available_rundirs]
self.render('rundirs.html',
pages=pages,
project_dir=self.config.project_dir,
models=models,
active_rundir=self.config.rundir)
def post(self):
rundir = self.get_argument('rundir', None)
if rundir is not None:
self.config.rundir = rundir
self.get()
class Summary(BaraddurRequestHandler):
@gen.coroutine
def get(self):
self.render('summary.html',
pages=pages,
problem=self.config.problem)
class Status(BaraddurRequestHandler): class Status(BaraddurRequestHandler):
...@@ -113,15 +192,17 @@ class Status(BaraddurRequestHandler): ...@@ -113,15 +192,17 @@ class Status(BaraddurRequestHandler):
def modify_document(self, doc): def modify_document(self, doc):
self.nmodels = 0 self.nmodels = 0
self.source = ColumnDataSource( self.source = ColumnDataSource()
data={'n': num.ndarray(0), self.source.add(num.ndarray(0, dtype=num.float32),
'gm': num.ndarray(0)}) 'target_mean')
self.source.add(num.ndarray(0, dtype=num.float32),
'niter')
self.update_misfits() self.update_misfits()
plot = figure(webgl=True, plot = figure(webgl=True,
x_axis_label='Iteration #', x_axis_label='Iteration #',
y_axis_label='Misfit') y_axis_label='Misfit')
plot.scatter('n', 'gm', plot.scatter('niter', 'target_mean',
source=self.source, alpha=.4) source=self.source, alpha=.4)
doc.add_root(plot) doc.add_root(plot)
...@@ -129,25 +210,22 @@ class Status(BaraddurRequestHandler): ...@@ -129,25 +210,22 @@ class Status(BaraddurRequestHandler):
@gen.coroutine @gen.coroutine
def update_misfits(self): def update_misfits(self):
mx, misfits = grond.core.load_problem_data( new_nmodels, new_data = self.config.model.get_misfits(
self.config.rundir, self.config.problem, skip_nmodels=self.nmodels,
skip_models=self.nmodels) keys=self.source.data.keys())
new_nmodels = mx.shape[0] self.source.stream(new_data)
fits = num.mean(misfits, axis=1)
self.source.stream(dict(gm=fits[:, 0],
n=num.arange(new_nmodels,
dtype=num.int) +
self.nmodels + 1))
self.nmodels += new_nmodels self.nmodels += new_nmodels
bokeh_handlers = {'misfit_plot': MisfitsPlot} bokeh_handlers = {'misfit': MisfitsPlot}
@gen.coroutine @gen.coroutine
def get(self): def get(self):
self.render('status.html', self.render('status.html',
pages=pages, pages=pages,
misfit_plot=autoload_server(None, url='/misfit_plot'), misfit_plot=autoload_server(
None,
app_path='/misfit',
url='plots'),
problem=self.config.problem) problem=self.config.problem)
...@@ -162,8 +240,9 @@ class Parameters(BaraddurRequestHandler): ...@@ -162,8 +240,9 @@ class Parameters(BaraddurRequestHandler):
problem = self.config.problem problem = self.config.problem
self.source = ColumnDataSource() self.source = ColumnDataSource()
for p in ['n'] + [p.name for p in problem.parameters]: for p in ['niter'] + [p.name for p in problem.parameters]:
self.source.add(num.ndarray(0), p) self.source.add(num.ndarray(0, dtype=num.float32), p)
self.model = GrondBokehModel(self.config.rundir)
self.update_parameters() self.update_parameters()
plots = [] plots = []
...@@ -171,11 +250,10 @@ class Parameters(BaraddurRequestHandler): ...@@ -171,11 +250,10 @@ class Parameters(BaraddurRequestHandler):
fig = figure(webgl=True, fig = figure(webgl=True,
x_axis_label='Iteration #', x_axis_label='Iteration #',
y_axis_label='%s [%s]' % (par.label, par.unit)) y_axis_label='%s [%s]' % (par.label, par.unit))
fig.scatter('n', par.name, fig.scatter('niter', par.name,
source=self.source, alpha=.4) source=self.source, alpha=.4)
plots.append(fig) plots.append(fig)
plots += [None] * (self.ncols - (len(plots) % self.ncols)) plots += [None] * (self.ncols - (len(plots) % self.ncols))
print plots
grid = layouts.gridplot( grid = layouts.gridplot(
plots, plots,
...@@ -187,44 +265,23 @@ class Parameters(BaraddurRequestHandler): ...@@ -187,44 +265,23 @@ class Parameters(BaraddurRequestHandler):
@gen.coroutine @gen.coroutine
def update_parameters(self): def update_parameters(self):
problem = self.config.problem new_nmodels, new_data = self.config.model.get_models(
skip_nmodels=self.nmodels,
try: keys=self.source.data.keys())
mx, misfits = grond.core.load_problem_data(
self.config.rundir, problem, skip_models=self.nmodels)
except IOError:
return
new_nmodels = mx.shape[0]
new_data = {}
for ip, par in enumerate(problem.parameters):
new_data[par.name] = mx[:, ip]
new_data['n'] = num.arange(new_nmodels, dtype=num.int) +\
self.nmodels + 1
self.source.stream(new_data) self.source.stream(new_data)
self.nmodels += new_nmodels self.nmodels += new_nmodels
bokeh_handlers = {'parameter_plot': ParameterPlots} bokeh_handlers = {'parameters': ParameterPlots}
@gen.coroutine @gen.coroutine
def get(self): def get(self):
self.render('parameter_plots.html', self.render('parameters.html',
pages=pages, pages=pages,
parameter_plot=autoload_server( parameter_plot=autoload_server(
None, None,
url='/parameter_plot'), app_path='/parameters',
problem=self.config.problem) url='plots'),
class Summary(BaraddurRequestHandler):
@gen.coroutine
def get(self):
self.render('summary.html',
pages=pages,
problem=self.config.problem) problem=self.config.problem)
...@@ -250,38 +307,36 @@ class Targets(BaraddurRequestHandler): ...@@ -250,38 +307,36 @@ class Targets(BaraddurRequestHandler):
self.config.rundir, self.config.problem, self.config.rundir, self.config.problem,
skip_models=self.nmodels) skip_models=self.nmodels)
print misfits
new_nmodels = mx.shape[0] new_nmodels = mx.shape[0]
# self.source.stream(dict(m=misfits[:, :, 0],
# n=num.arange(new_nmodels,
# dtype=num.int) +
# self.nmodels + 1))
self.nmodels += new_nmodels self.nmodels += new_nmodels
bokeh_handlers = {'contribution_plot': TargetContributionPlot} bokeh_handlers = {'contributions': TargetContributionPlot}
@gen.coroutine @gen.coroutine
def get(self): def get(self):
self.render('targets.html', self.render('targets.html',
contribution_plot=autoload_server( contribution_plot=autoload_server(
None, None,
url='/contribution_plot'), app_path='/contributions',
url='plots'),
pages=pages, pages=pages,
problem=self.config.problem) problem=self.config.problem)
pages = OrderedDict([ pages = OrderedDict([
('Rundirs', Rundirs),
('Summary', Summary), ('Summary', Summary),
('Status', Status), ('Status', Status),
('Parameters', Parameters), ('Parameters', Parameters),
('Targets', Targets), # ('Targets', Targets),
]) ])