server.py 15 KB
Newer Older
marius's avatar
marius committed
1
2
import tornado.ioloop
import grond
marius's avatar
wip    
marius committed
3
import os
Marius Isken's avatar
Marius Isken committed
4
import glob
marius's avatar
marius committed
5
import os.path as op
Marius Isken's avatar
Marius Isken committed
6
import logging
7
import numpy as num
Marius Isken's avatar
wip    
Marius Isken committed
8
import socket
marius's avatar
marius committed
9
10

from collections import OrderedDict
Marius Isken's avatar
wip    
Marius Isken committed
11
from datetime import datetime
marius's avatar
marius committed
12

Marius Isken's avatar
wip    
Marius Isken committed
13
from pyrocko.guts import Object, Bool, String, Int, List
14

Marius Isken's avatar
Marius Isken committed
15
from tornado.web import RequestHandler, StaticFileHandler
marius's avatar
marius committed
16
17
18
from tornado import gen

from bokeh.embed import autoload_server
Marius Isken's avatar
Marius Isken committed
19
20
21
from bokeh.application import Application
from bokeh.server.server import Server as BokehServer
from bokeh.application.handlers import Handler as BokehHandler
marius's avatar
marius committed
22

Marius Isken's avatar
Marius Isken committed
23
24
from bokeh.models import ColumnDataSource
from bokeh import layouts
marius's avatar
marius committed
25
26
from bokeh.plotting import figure

Marius Isken's avatar
wip    
Marius Isken committed
27
28
29
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('grond.baraddur')

marius's avatar
marius committed
30

Marius Isken's avatar
Marius Isken committed
31
32
33
34
35
36
37
38
39
def makeColorGradient(misfits, fr=1., fg=.5, fb=1.,
                      pr=0, pg=2.5, pb=4):
    misfits /= misfits.max()
    r = num.sin(fr * misfits + pr) * 127 + 128
    g = num.sin(fg * misfits + pg) * 127 + 128
    b = num.sin(fb * misfits + pb) * 127 + 128
    return ['#%02x%02x%02x' % (r[i], g[i], b[i]) for i in xrange(misfits.size)]


40
41
42
43
class BaraddurRequestHandler(RequestHandler):
    def initialize(self, config):
        self.config = config

Marius Isken's avatar
Marius Isken committed
44

45
46
47
48
49
50
class BaraddurBokehHandler(BokehHandler):
    def __init__(self, config, *args, **kwargs):
        BokehHandler.__init__(self, *args, **kwargs)
        self.config = config


Marius Isken's avatar
wip    
Marius Isken committed
51
class BaraddurModel(object):
Marius Isken's avatar
Marius Isken committed
52
53
    def __init__(self, rundir):
        self.set_rundir(rundir)
marius's avatar
wip    
marius committed
54
55
56

    def set_rundir(self, rundir):
        logger.debug('Loading problem from %s' % rundir)
Marius Isken's avatar
Marius Isken committed
57
        self.rundir = op.abspath(rundir)
marius's avatar
wip    
marius committed
58
59
60
61
62
        self.problem = grond.core.load_problem_info(self.rundir)
        self.parameters = self.problem.parameters
        self.nparameters = self.problem.nparameters
        self.ntargets = self.problem.ntargets

Marius Isken's avatar
Marius Isken committed
63
64
65
66
67
    def _jp(self, file):
        return op.join(self.rundir, file)

    @property
    def start_time(self):
Marius Isken's avatar
wip    
Marius Isken committed
68
69
70
71
72
73
        return datetime.fromtimestamp(os.stat(self.rundir).st_mtime)

    @property
    def duration(self):
        return datetime.fromtimestamp(
            os.stat(self._jp('models')).st_ctime) - self.start_time
Marius Isken's avatar
Marius Isken committed
74
75
76
77
78
79
80
81
82
83
84
85
86
87

    @property
    def name(self):
        return op.split(self.rundir)[-1]

    @property
    def best_misfit(self):
        _, data = self.get_misfits(keys='target_mean')
        return data['target_mean'].min()

    @property
    def niterations(self):
        return op.getsize(self._jp('models')) / (self.nparameters * 8)

Marius Isken's avatar
wip    
Marius Isken committed
88
89
90
91
    @property
    def iter_per_second(self):
        return self.niterations / self.duration.seconds

Marius Isken's avatar
Marius Isken committed
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
    @staticmethod
    def validate_rundir(rundir):

        def jp(fn):
            return op.join(rundir, fn)

        for fn in ['models', 'misfits', 'problem.yaml']:
            if op.lexists(jp(fn)):
                if op.getsize(jp(fn)) == 0:
                    return False
            else:
                return False
        return True

    def get_models(self, skip_nmodels=0, keys=None):
        fn = self._jp('models')

marius's avatar
wip    
marius committed
109
110
111
112
113
114
        with open(fn, 'r') as f:
            nmodels = os.fstat(f.fileno()).st_size / (self.nparameters * 8)
            nmodels -= skip_nmodels
            f.seek(skip_nmodels * self.nparameters * 8)
            data = num.fromfile(
                f, dtype='<f8', count=nmodels * self.nparameters)\
Marius Isken's avatar
Marius Isken committed
115
                .astype(num.float32)
marius's avatar
wip    
marius committed
116

Marius Isken's avatar
wip    
Marius Isken committed
117
        nmodels = data.size/self.nparameters
marius's avatar
wip    
marius committed
118
119
        models = data.reshape((nmodels, self.nparameters))

Marius Isken's avatar
wip    
Marius Isken committed
120
        models_dict = {}
marius's avatar
wip    
marius committed
121
        for ip, par in enumerate(self.parameters):
Marius Isken's avatar
wip    
Marius Isken committed
122
123
            models_dict[par.name] = models[:, ip]
        models_dict['niter'] = num.arange(nmodels, dtype=num.int)\
Marius Isken's avatar
Marius Isken committed
124
125
126
            + skip_nmodels + 1

        if keys is not None:
Marius Isken's avatar
wip    
Marius Isken committed
127
            for k in models_dict.keys():
Marius Isken's avatar
Marius Isken committed
128
                if k not in keys:
Marius Isken's avatar
wip    
Marius Isken committed
129
                    models_dict.pop(k)
Marius Isken's avatar
Marius Isken committed
130

Marius Isken's avatar
wip    
Marius Isken committed
131
        return nmodels, models_dict
marius's avatar
wip    
marius committed
132

Marius Isken's avatar
Marius Isken committed
133
134
    def get_misfits(self, skip_nmodels=0, keys=None):
        fn = self._jp('misfits')
marius's avatar
wip    
marius committed
135
136

        with open(fn, 'r') as f:
Marius Isken's avatar
wip    
Marius Isken committed
137
            nmodels = os.fstat(f.fileno()).st_size / (self.ntargets * 2 * 8)
marius's avatar
wip    
marius committed
138
139
140
141
            nmodels -= skip_nmodels
            f.seek(skip_nmodels * self.ntargets * 2 * 8)
            data = num.fromfile(
                f, dtype='<f8', count=nmodels*self.ntargets*2)\
Marius Isken's avatar
Marius Isken committed
142
                .astype(num.float32)
marius's avatar
wip    
marius committed
143

Marius Isken's avatar
wip    
Marius Isken committed
144
145
        nmodels = data.size/(self.ntargets * 2)

marius's avatar
wip    
marius committed
146
147
148
149
150
151
152
153
154
        data = data.reshape((nmodels, self.ntargets*2))
        combi = num.empty_like(data)
        combi[:, 0::2] = data[:, :self.ntargets]
        combi[:, 1::2] = data[:, self.ntargets:]
        misfits = combi.reshape((nmodels, self.ntargets, 2))

        mf_dict = {}
        for it in xrange(self.ntargets):
            mf_dict['target_%03d' % it] = misfits[:, it, 0]
Marius Isken's avatar
Marius Isken committed
155
        mf_dict['target_mean'] = num.mean(misfits, axis=1)[:, 0]
marius's avatar
wip    
marius committed
156

Marius Isken's avatar
Marius Isken committed
157
158
159
160
161
162
163
164
165
166
167
168
169
170
        mf_dict['niter'] = num.arange(nmodels, dtype=num.int)\
            + skip_nmodels + 1

        if keys is not None:
            for k in mf_dict.keys():
                if k not in keys:
                    mf_dict.pop(k)

        return nmodels, mf_dict


class Rundirs(BaraddurRequestHandler):

    def get(self):
Marius Isken's avatar
wip    
Marius Isken committed
171
        models = [BaraddurModel(rundir=rd)
Marius Isken's avatar
Marius Isken committed
172
173
174
175
176
177
178
179
180
181
182
183
184
185
                  for rd in self.config.available_rundirs]
        self.render('rundirs.html',
                    pages=pages,
                    project_dir=self.config.project_dir,
                    models=models,
                    active_rundir=self.config.rundir)

    def post(self):
        rundir = self.get_argument('rundir', None)
        if rundir is not None:
            self.config.rundir = rundir
        self.get()


Marius Isken's avatar
wip    
Marius Isken committed
186
class Misfit(BaraddurRequestHandler):
Marius Isken's avatar
Marius Isken committed
187
188
189
190
191
192

    @gen.coroutine
    def get(self):
        self.render('summary.html',
                    pages=pages,
                    problem=self.config.problem)
marius's avatar
wip    
marius committed
193
194


195
196
197
class Status(BaraddurRequestHandler):

    class MisfitsPlot(BaraddurBokehHandler):
Marius Isken's avatar
Marius Isken committed
198
199

        def modify_document(self, doc):
200
            self.nmodels = 0
Marius Isken's avatar
Marius Isken committed
201
202
203
204
205
            self.source = ColumnDataSource()
            self.source.add(num.ndarray(0, dtype=num.float32),
                            'target_mean')
            self.source.add(num.ndarray(0, dtype=num.float32),
                            'niter')
Marius Isken's avatar
Marius Isken committed
206
207
208
209
210
            self.update_misfits()

            plot = figure(webgl=True,
                          x_axis_label='Iteration #',
                          y_axis_label='Misfit')
Marius Isken's avatar
Marius Isken committed
211
            plot.scatter('niter', 'target_mean',
Marius Isken's avatar
Marius Isken committed
212
213
214
215
216
217
218
                         source=self.source, alpha=.4)

            doc.add_root(plot)
            doc.add_periodic_callback(self.update_misfits, 1e3)

        @gen.coroutine
        def update_misfits(self):
Marius Isken's avatar
Marius Isken committed
219
220
221
222
            new_nmodels, new_data = self.config.model.get_misfits(
                skip_nmodels=self.nmodels,
                keys=self.source.data.keys())
            self.source.stream(new_data)
223
            self.nmodels += new_nmodels
Marius Isken's avatar
Marius Isken committed
224

Marius Isken's avatar
Marius Isken committed
225
    bokeh_handlers = {'misfit': MisfitsPlot}
Marius Isken's avatar
Marius Isken committed
226
227
228
229
230

    @gen.coroutine
    def get(self):
        self.render('status.html',
                    pages=pages,
Marius Isken's avatar
Marius Isken committed
231
232
233
234
                    misfit_plot=autoload_server(
                        None,
                        app_path='/misfit',
                        url='plots'),
235
                    problem=self.config.problem)
Marius Isken's avatar
Marius Isken committed
236
237


238
class Parameters(BaraddurRequestHandler):
Marius Isken's avatar
Marius Isken committed
239

240
    class ParameterPlots(BaraddurBokehHandler):
Marius Isken's avatar
Marius Isken committed
241
242
243
244

        ncols = 4

        def modify_document(self, doc):
245
            self.nmodels = 0
246
247
            problem = self.config.problem

248
            self.source = ColumnDataSource()
Marius Isken's avatar
Marius Isken committed
249
250
            for p in ['niter'] + [p.name for p in problem.parameters]:
                self.source.add(num.ndarray(0, dtype=num.float32), p)
Marius Isken's avatar
wip    
Marius Isken committed
251
            self.model = BaraddurModel(self.config.rundir)
Marius Isken's avatar
Marius Isken committed
252
253
254
255
256
257
258
            self.update_parameters()

            plots = []
            for par in problem.parameters:
                fig = figure(webgl=True,
                             x_axis_label='Iteration #',
                             y_axis_label='%s [%s]' % (par.label, par.unit))
Marius Isken's avatar
Marius Isken committed
259
                fig.scatter('niter', par.name,
260
                            source=self.source, alpha=.4)
Marius Isken's avatar
Marius Isken committed
261
                plots.append(fig)
Marius Isken's avatar
wip    
Marius Isken committed
262
            plots += [None] * (self.ncols - (len(plots) % self.ncols))
Marius Isken's avatar
Marius Isken committed
263
264
265
266
267
268
269

            grid = layouts.gridplot(
                plots,
                responsive=True,
                ncols=self.ncols)

            doc.add_root(grid)
270
            doc.add_periodic_callback(self.update_parameters, 2.5*1e3)
Marius Isken's avatar
Marius Isken committed
271
272
273

        @gen.coroutine
        def update_parameters(self):
Marius Isken's avatar
Marius Isken committed
274
275
276
            new_nmodels, new_data = self.config.model.get_models(
                skip_nmodels=self.nmodels,
                keys=self.source.data.keys())
277
278
            self.source.stream(new_data)
            self.nmodels += new_nmodels
Marius Isken's avatar
Marius Isken committed
279

Marius Isken's avatar
Marius Isken committed
280
    bokeh_handlers = {'parameters': ParameterPlots}
marius's avatar
marius committed
281
282
283
284

    @gen.coroutine
    def get(self):

Marius Isken's avatar
Marius Isken committed
285
        self.render('parameters.html',
Marius Isken's avatar
Marius Isken committed
286
287
288
                    pages=pages,
                    parameter_plot=autoload_server(
                        None,
Marius Isken's avatar
Marius Isken committed
289
290
                        app_path='/parameters',
                        url='plots'),
291
                    problem=self.config.problem)
marius's avatar
marius committed
292
293


Marius Isken's avatar
wip    
Marius Isken committed
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
class Targets(BaraddurRequestHandler):

    class TargetContributionPlot(BaraddurBokehHandler):

        def modify_document(self, doc):
            self.nmodels = 0
            self.source = ColumnDataSource()
            self.update_contributions()

            plot = figure(webgl=True,
                          x_axis_label='Iteration #',
                          y_axis_label='Misfit')

            doc.add_root(plot)
            doc.add_periodic_callback(self.update_contributions, 1e3)

        @gen.coroutine
        def update_contributions(self):
            mx, misfits = grond.core.load_problem_data(
                self.config.rundir, self.config.problem,
                skip_models=self.nmodels)

            new_nmodels = mx.shape[0]
            self.nmodels += new_nmodels

Marius Isken's avatar
Marius Isken committed
319
    bokeh_handlers = {'contributions': TargetContributionPlot}
Marius Isken's avatar
Marius Isken committed
320

marius's avatar
marius committed
321
322
    @gen.coroutine
    def get(self):
Marius Isken's avatar
wip    
Marius Isken committed
323
324
325
        self.render('targets.html',
                    contribution_plot=autoload_server(
                        None,
Marius Isken's avatar
Marius Isken committed
326
327
                        app_path='/contributions',
                        url='plots'),
Marius Isken's avatar
wip    
Marius Isken committed
328
329
                    pages=pages,
                    problem=self.config.problem)
marius's avatar
marius committed
330
331
332


pages = OrderedDict([
Marius Isken's avatar
Marius Isken committed
333
    ('Rundirs', Rundirs),
Marius Isken's avatar
wip    
Marius Isken committed
334
335
    ('Summary', Misfit),
    ('Misfit', Status),
Marius Isken's avatar
Marius Isken committed
336
    ('Parameters', Parameters),
Marius Isken's avatar
Marius Isken committed
337
    # ('Targets', Targets),
marius's avatar
marius committed
338
339
340
])


341
class BaraddurConfig(Object):
Marius Isken's avatar
Marius Isken committed
342
343
344
345
    rundir = String.T()
    project_dir = String.T(
        help='Grond project dir.',
        optional=True)
346
    debug = Bool.T(
347
        default=False,
348
        optional=True)
Marius Isken's avatar
wip    
Marius Isken committed
349
350
351
352
    hosts = List.T(
        String.T(),
        default=['*'],
        optional=True,
marius's avatar
wip    
marius committed
353
        help='List of allowed hosts, default is all \'*\'.')
marius's avatar
wip    
marius committed
354
355
    port = Int.T(
        default=8080,
Marius Isken's avatar
wip    
Marius Isken committed
356
357
        optional=True,
        help='Port to listen on.')
358

Marius Isken's avatar
Marius Isken committed
359
360
361
362
363
    def __init__(self, *args, **kwargs):
        Object.__init__(self, *args, **kwargs)
        self._rundir = None
        self._model = None

364
365
366
367
    @property
    def problem(self):
        return grond.core.load_problem_info(self.rundir)

Marius Isken's avatar
Marius Isken committed
368
369
370
371
372
373
374
375
376
377
378
379
380
381
    @property
    def rundir(self):
        return self._rundir

    @rundir.getter
    def rundir(self):
        if self._rundir is None:
            if len(self.available_rundirs) > 0:
                self.rundir = self.available_rundirs[0]
        return self._rundir

    @rundir.setter
    def rundir(self, value):
        self._rundir = value
Marius Isken's avatar
wip    
Marius Isken committed
382
        self._model = BaraddurModel(rundir=self._rundir)
Marius Isken's avatar
Marius Isken committed
383
384
385
386

    @property
    def model(self):
        if self._model is None:
Marius Isken's avatar
wip    
Marius Isken committed
387
            self._model = BaraddurModel(rundir=self.rundir)
Marius Isken's avatar
Marius Isken committed
388
389
390
391
392
393
394
        return self._model

    @property
    def available_rundirs(self):
        rundirs = []
        for d in [d for d in glob.glob(op.join(self.project_dir, '*'))
                  if op.isdir(d)]:
Marius Isken's avatar
wip    
Marius Isken committed
395
            if BaraddurModel.validate_rundir(d):
Marius Isken's avatar
Marius Isken committed
396
397
398
                rundirs.append(d)
        return rundirs

399
400

class Baraddur(BokehServer):
Marius Isken's avatar
Marius Isken committed
401
402
403
404
405
406
407
408

    def __init__(self, config=None, project_dir=None, *args, **kwargs):
        if config is not None:
            self.config = config
        elif project_dir is not None:
            self.config = BaraddurConfig(project_dir=project_dir)
        else:
            raise AttributeError('No project dir set!')
409
        self.ioloop = tornado.ioloop.IOLoop.current()
Marius Isken's avatar
wip    
Marius Isken committed
410
        port_offset = 0
411

Marius Isken's avatar
wip    
Marius Isken committed
412
413
414
415
416
        while True:
            try:
                BokehServer.__init__(
                    self,
                    self.get_bokeh_apps(),
417
                    io_loop=self.ioloop,
Marius Isken's avatar
wip    
Marius Isken committed
418
419
420
421
422
423
424
                    extra_patterns=self.get_tornado_handlers(),
                    port=self.config.port + port_offset,
                    host=self.config.hosts)
                break
            except socket.error as se:
                if se.errno == 98 and port_offset < 50:  # Port in use
                    port_offset += 1
Marius Isken's avatar
Marius Isken committed
425
426
427
                    logger.warning('Port %d in use, bailing to %d'
                                   % (self.config.port + port_offset - 1,
                                      self.config.port + port_offset))
Marius Isken's avatar
wip    
Marius Isken committed
428
429
                else:
                    raise se
430
        tornado_app = self._tornado
431
432
        tornado_app.settings['template_path'] = op.join(
            op.dirname(op.abspath(__file__)), 'templates')
433
434
435
436
437
438
439
440
441
442

        if self.config.debug:
            tornado_app.settings.setdefault('autoreload', True)
            tornado_app.settings.setdefault('compiled_template_cache', False)
            tornado_app.settings.setdefault('static_hash_cache', False)
            tornado_app.settings.setdefault('serve_traceback', True)
            # Automatically reload modified modules
            from tornado import autoreload
            autoreload.start()

Marius Isken's avatar
wip    
Marius Isken committed
443
            logging.getLogger('').setLevel(logging.DEBUG)
444
445
446
447
448
449
450
451
452
453

    def get_bokeh_apps(self):
        bokeh_apps = {}
        for tornado_handler in pages.itervalues():
            handler_docs = getattr(tornado_handler, 'bokeh_handlers',
                                   None)
            if handler_docs is None:
                continue

            for url, bokeh_handler in handler_docs.iteritems():
Marius Isken's avatar
Marius Isken committed
454
                bokeh_apps['/plots/%s' % url] = Application(bokeh_handler(
455
456
457
458
                    self.config))
        return bokeh_apps

    def get_tornado_handlers(self):
Marius Isken's avatar
Marius Isken committed
459
460
        return [(r'/', pages.values()[0],
                 {'config': self.config})] +\
461
462
               [(r'/%s' % title, handler,
                 {'config': self.config})
Marius Isken's avatar
Marius Isken committed
463
                for title, handler in pages.iteritems()] +\
Marius Isken's avatar
Marius Isken committed
464
465
               [(r'/res/(.*)', StaticFileHandler,
                {'path': op.join(op.dirname(__file__), 'res')})]
Marius Isken's avatar
Marius Isken committed
466

467
    def start(self, signal=None):
Marius Isken's avatar
Marius Isken committed
468
469
        logger.info('Starting Baraddur server on http://%s:%d'
                    % ('localhost', self.port))
470
471
472
473
474
475
476

        if signal is not None:
            def shutdown():
                if not signal.empty():
                    self.stop()
            tornado.ioloop.PeriodicCallback(shutdown, 2000).start()

477
        BokehServer.start(self)
478
479
480
481
482
483
        self.ioloop.start()

    def stop(self, *args, **kwargs):
        logger.info('Stopping Baraddur server...')
        BokehServer.stop(self)
        self.ioloop.stop()
484
485


Marius Isken's avatar
Marius Isken committed
486
if __name__ == '__main__':
487
488
489
    baraddur = Baraddur(
        rundir='/home/marius/Development/testing/grond/rundir')
    baraddur.start()