server.py 15.1 KB
Newer Older
marius's avatar
marius committed
1
2
import tornado.ioloop
import grond
marius's avatar
wip    
marius committed
3
import os
Marius Isken's avatar
Marius Isken committed
4
import glob
marius's avatar
marius committed
5
import os.path as op
Marius Isken's avatar
Marius Isken committed
6
import logging
7
import numpy as num
Marius Isken's avatar
wip    
Marius Isken committed
8
import socket
marius's avatar
marius committed
9
10

from collections import OrderedDict
Marius Isken's avatar
wip    
Marius Isken committed
11
from datetime import datetime
marius's avatar
marius committed
12

Marius Isken's avatar
wip    
Marius Isken committed
13
from pyrocko.guts import Object, Bool, String, Int, List
14

Marius Isken's avatar
Marius Isken committed
15
from tornado.web import RequestHandler, StaticFileHandler
marius's avatar
marius committed
16
17
18
from tornado import gen

from bokeh.embed import autoload_server
Marius Isken's avatar
Marius Isken committed
19
20
21
from bokeh.application import Application
from bokeh.server.server import Server as BokehServer
from bokeh.application.handlers import Handler as BokehHandler
marius's avatar
marius committed
22

Marius Isken's avatar
Marius Isken committed
23
24
from bokeh.models import ColumnDataSource
from bokeh import layouts
marius's avatar
marius committed
25
26
from bokeh.plotting import figure

Marius Isken's avatar
wip    
Marius Isken committed
27
28
29
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('grond.baraddur')

marius's avatar
marius committed
30

Marius Isken's avatar
Marius Isken committed
31
32
33
34
35
36
37
38
39
def makeColorGradient(misfits, fr=1., fg=.5, fb=1.,
                      pr=0, pg=2.5, pb=4):
    misfits /= misfits.max()
    r = num.sin(fr * misfits + pr) * 127 + 128
    g = num.sin(fg * misfits + pg) * 127 + 128
    b = num.sin(fb * misfits + pb) * 127 + 128
    return ['#%02x%02x%02x' % (r[i], g[i], b[i]) for i in xrange(misfits.size)]


40
41
42
43
class BaraddurRequestHandler(RequestHandler):
    def initialize(self, config):
        self.config = config

Marius Isken's avatar
Marius Isken committed
44

45
46
47
48
49
50
class BaraddurBokehHandler(BokehHandler):
    def __init__(self, config, *args, **kwargs):
        BokehHandler.__init__(self, *args, **kwargs)
        self.config = config


Marius Isken's avatar
wip    
Marius Isken committed
51
class BaraddurModel(object):
Marius Isken's avatar
Marius Isken committed
52
53
    def __init__(self, rundir):
        self.set_rundir(rundir)
marius's avatar
wip    
marius committed
54
55
56

    def set_rundir(self, rundir):
        logger.debug('Loading problem from %s' % rundir)
Marius Isken's avatar
Marius Isken committed
57
        self.rundir = op.abspath(rundir)
marius's avatar
wip    
marius committed
58
59
60
61
62
        self.problem = grond.core.load_problem_info(self.rundir)
        self.parameters = self.problem.parameters
        self.nparameters = self.problem.nparameters
        self.ntargets = self.problem.ntargets

Marius Isken's avatar
Marius Isken committed
63
64
65
66
67
    def _jp(self, file):
        return op.join(self.rundir, file)

    @property
    def start_time(self):
Marius Isken's avatar
wip    
Marius Isken committed
68
69
70
71
72
73
        return datetime.fromtimestamp(os.stat(self.rundir).st_mtime)

    @property
    def duration(self):
        return datetime.fromtimestamp(
            os.stat(self._jp('models')).st_ctime) - self.start_time
Marius Isken's avatar
Marius Isken committed
74
75
76
77
78
79
80
81
82
83
84
85
86
87

    @property
    def name(self):
        return op.split(self.rundir)[-1]

    @property
    def best_misfit(self):
        _, data = self.get_misfits(keys='target_mean')
        return data['target_mean'].min()

    @property
    def niterations(self):
        return op.getsize(self._jp('models')) / (self.nparameters * 8)

Marius Isken's avatar
wip    
Marius Isken committed
88
89
90
91
    @property
    def iter_per_second(self):
        return self.niterations / self.duration.seconds

Marius Isken's avatar
Marius Isken committed
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
    @staticmethod
    def validate_rundir(rundir):

        def jp(fn):
            return op.join(rundir, fn)

        for fn in ['models', 'misfits', 'problem.yaml']:
            if op.lexists(jp(fn)):
                if op.getsize(jp(fn)) == 0:
                    return False
            else:
                return False
        return True

    def get_models(self, skip_nmodels=0, keys=None):
        fn = self._jp('models')

marius's avatar
wip    
marius committed
109
110
111
112
113
114
        with open(fn, 'r') as f:
            nmodels = os.fstat(f.fileno()).st_size / (self.nparameters * 8)
            nmodels -= skip_nmodels
            f.seek(skip_nmodels * self.nparameters * 8)
            data = num.fromfile(
                f, dtype='<f8', count=nmodels * self.nparameters)\
Marius Isken's avatar
Marius Isken committed
115
                .astype(num.float32)
marius's avatar
wip    
marius committed
116

Marius Isken's avatar
wip    
Marius Isken committed
117
        nmodels = data.size/self.nparameters
marius's avatar
wip    
marius committed
118
119
        models = data.reshape((nmodels, self.nparameters))

Marius Isken's avatar
wip    
Marius Isken committed
120
        models_dict = {}
marius's avatar
wip    
marius committed
121
        for ip, par in enumerate(self.parameters):
Marius Isken's avatar
wip    
Marius Isken committed
122
123
            models_dict[par.name] = models[:, ip]
        models_dict['niter'] = num.arange(nmodels, dtype=num.int)\
Marius Isken's avatar
Marius Isken committed
124
125
126
            + skip_nmodels + 1

        if keys is not None:
Marius Isken's avatar
wip    
Marius Isken committed
127
            for k in models_dict.keys():
Marius Isken's avatar
Marius Isken committed
128
                if k not in keys:
Marius Isken's avatar
wip    
Marius Isken committed
129
                    models_dict.pop(k)
Marius Isken's avatar
Marius Isken committed
130

Marius Isken's avatar
wip    
Marius Isken committed
131
        return nmodels, models_dict
marius's avatar
wip    
marius committed
132

Marius Isken's avatar
Marius Isken committed
133
134
    def get_misfits(self, skip_nmodels=0, keys=None):
        fn = self._jp('misfits')
marius's avatar
wip    
marius committed
135
136

        with open(fn, 'r') as f:
Marius Isken's avatar
wip    
Marius Isken committed
137
            nmodels = os.fstat(f.fileno()).st_size / (self.ntargets * 2 * 8)
marius's avatar
wip    
marius committed
138
139
140
141
            nmodels -= skip_nmodels
            f.seek(skip_nmodels * self.ntargets * 2 * 8)
            data = num.fromfile(
                f, dtype='<f8', count=nmodels*self.ntargets*2)\
Marius Isken's avatar
Marius Isken committed
142
                .astype(num.float32)
marius's avatar
wip    
marius committed
143

Marius Isken's avatar
wip    
Marius Isken committed
144
145
        nmodels = data.size/(self.ntargets * 2)

marius's avatar
wip    
marius committed
146
147
148
149
150
151
152
153
154
        data = data.reshape((nmodels, self.ntargets*2))
        combi = num.empty_like(data)
        combi[:, 0::2] = data[:, :self.ntargets]
        combi[:, 1::2] = data[:, self.ntargets:]
        misfits = combi.reshape((nmodels, self.ntargets, 2))

        mf_dict = {}
        for it in xrange(self.ntargets):
            mf_dict['target_%03d' % it] = misfits[:, it, 0]
Marius Isken's avatar
Marius Isken committed
155
        mf_dict['target_mean'] = num.mean(misfits, axis=1)[:, 0]
marius's avatar
wip    
marius committed
156

Marius Isken's avatar
Marius Isken committed
157
158
159
160
161
162
163
164
165
166
167
168
169
170
        mf_dict['niter'] = num.arange(nmodels, dtype=num.int)\
            + skip_nmodels + 1

        if keys is not None:
            for k in mf_dict.keys():
                if k not in keys:
                    mf_dict.pop(k)

        return nmodels, mf_dict


class Rundirs(BaraddurRequestHandler):

    def get(self):
Marius Isken's avatar
wip    
Marius Isken committed
171
        models = [BaraddurModel(rundir=rd)
Marius Isken's avatar
Marius Isken committed
172
173
174
175
176
177
178
179
180
181
182
183
184
185
                  for rd in self.config.available_rundirs]
        self.render('rundirs.html',
                    pages=pages,
                    project_dir=self.config.project_dir,
                    models=models,
                    active_rundir=self.config.rundir)

    def post(self):
        rundir = self.get_argument('rundir', None)
        if rundir is not None:
            self.config.rundir = rundir
        self.get()


Marius Isken's avatar
wip    
Marius Isken committed
186
class Summary(BaraddurRequestHandler):
Marius Isken's avatar
Marius Isken committed
187
188
189
190
191
192

    @gen.coroutine
    def get(self):
        self.render('summary.html',
                    pages=pages,
                    problem=self.config.problem)
marius's avatar
wip    
marius committed
193
194


Marius Isken's avatar
wip    
Marius Isken committed
195
class Misfit(BaraddurRequestHandler):
196
197

    class MisfitsPlot(BaraddurBokehHandler):
Marius Isken's avatar
Marius Isken committed
198
199

        def modify_document(self, doc):
200
            self.nmodels = 0
Marius Isken's avatar
Marius Isken committed
201
202
203
204
205
            self.source = ColumnDataSource()
            self.source.add(num.ndarray(0, dtype=num.float32),
                            'target_mean')
            self.source.add(num.ndarray(0, dtype=num.float32),
                            'niter')
Marius Isken's avatar
Marius Isken committed
206
207
208
            self.update_misfits()

            plot = figure(webgl=True,
Marius Isken's avatar
wip    
Marius Isken committed
209
210
                          responsive=True,
                          plot_height=300,
Marius Isken's avatar
Marius Isken committed
211
212
                          x_axis_label='Iteration #',
                          y_axis_label='Misfit')
Marius Isken's avatar
Marius Isken committed
213
            plot.scatter('niter', 'target_mean',
Marius Isken's avatar
Marius Isken committed
214
215
216
217
218
219
220
                         source=self.source, alpha=.4)

            doc.add_root(plot)
            doc.add_periodic_callback(self.update_misfits, 1e3)

        @gen.coroutine
        def update_misfits(self):
Marius Isken's avatar
Marius Isken committed
221
222
223
224
            new_nmodels, new_data = self.config.model.get_misfits(
                skip_nmodels=self.nmodels,
                keys=self.source.data.keys())
            self.source.stream(new_data)
225
            self.nmodels += new_nmodels
Marius Isken's avatar
Marius Isken committed
226

Marius Isken's avatar
Marius Isken committed
227
    bokeh_handlers = {'misfit': MisfitsPlot}
Marius Isken's avatar
Marius Isken committed
228
229
230

    @gen.coroutine
    def get(self):
Marius Isken's avatar
wip    
Marius Isken committed
231
        self.render('misfit.html',
Marius Isken's avatar
Marius Isken committed
232
                    pages=pages,
Marius Isken's avatar
Marius Isken committed
233
234
235
236
                    misfit_plot=autoload_server(
                        None,
                        app_path='/misfit',
                        url='plots'),
237
                    problem=self.config.problem)
Marius Isken's avatar
Marius Isken committed
238
239


240
class Parameters(BaraddurRequestHandler):
Marius Isken's avatar
Marius Isken committed
241

242
    class ParameterPlots(BaraddurBokehHandler):
Marius Isken's avatar
Marius Isken committed
243
244
245
246

        ncols = 4

        def modify_document(self, doc):
247
            self.nmodels = 0
248
249
            problem = self.config.problem

250
            self.source = ColumnDataSource()
Marius Isken's avatar
Marius Isken committed
251
252
            for p in ['niter'] + [p.name for p in problem.parameters]:
                self.source.add(num.ndarray(0, dtype=num.float32), p)
Marius Isken's avatar
wip    
Marius Isken committed
253
            self.model = BaraddurModel(self.config.rundir)
Marius Isken's avatar
Marius Isken committed
254
255
256
257
258
            self.update_parameters()

            plots = []
            for par in problem.parameters:
                fig = figure(webgl=True,
Marius Isken's avatar
wip    
Marius Isken committed
259
                             responsive=True,
Marius Isken's avatar
Marius Isken committed
260
261
                             x_axis_label='Iteration #',
                             y_axis_label='%s [%s]' % (par.label, par.unit))
Marius Isken's avatar
Marius Isken committed
262
                fig.scatter('niter', par.name,
263
                            source=self.source, alpha=.4)
Marius Isken's avatar
Marius Isken committed
264
                plots.append(fig)
Marius Isken's avatar
wip    
Marius Isken committed
265
            plots += [None] * (self.ncols - (len(plots) % self.ncols))
Marius Isken's avatar
Marius Isken committed
266
267
268
269
270
271
272

            grid = layouts.gridplot(
                plots,
                responsive=True,
                ncols=self.ncols)

            doc.add_root(grid)
273
            doc.add_periodic_callback(self.update_parameters, 2.5*1e3)
Marius Isken's avatar
Marius Isken committed
274
275
276

        @gen.coroutine
        def update_parameters(self):
Marius Isken's avatar
Marius Isken committed
277
278
279
            new_nmodels, new_data = self.config.model.get_models(
                skip_nmodels=self.nmodels,
                keys=self.source.data.keys())
280
281
            self.source.stream(new_data)
            self.nmodels += new_nmodels
Marius Isken's avatar
Marius Isken committed
282

Marius Isken's avatar
Marius Isken committed
283
    bokeh_handlers = {'parameters': ParameterPlots}
marius's avatar
marius committed
284
285
286
287

    @gen.coroutine
    def get(self):

Marius Isken's avatar
Marius Isken committed
288
        self.render('parameters.html',
Marius Isken's avatar
Marius Isken committed
289
290
291
                    pages=pages,
                    parameter_plot=autoload_server(
                        None,
Marius Isken's avatar
Marius Isken committed
292
293
                        app_path='/parameters',
                        url='plots'),
294
                    problem=self.config.problem)
marius's avatar
marius committed
295
296


Marius Isken's avatar
wip    
Marius Isken committed
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
class Targets(BaraddurRequestHandler):

    class TargetContributionPlot(BaraddurBokehHandler):

        def modify_document(self, doc):
            self.nmodels = 0
            self.source = ColumnDataSource()
            self.update_contributions()

            plot = figure(webgl=True,
                          x_axis_label='Iteration #',
                          y_axis_label='Misfit')

            doc.add_root(plot)
            doc.add_periodic_callback(self.update_contributions, 1e3)

        @gen.coroutine
        def update_contributions(self):
            mx, misfits = grond.core.load_problem_data(
                self.config.rundir, self.config.problem,
                skip_models=self.nmodels)

            new_nmodels = mx.shape[0]
            self.nmodels += new_nmodels

Marius Isken's avatar
Marius Isken committed
322
    bokeh_handlers = {'contributions': TargetContributionPlot}
Marius Isken's avatar
Marius Isken committed
323

marius's avatar
marius committed
324
325
    @gen.coroutine
    def get(self):
Marius Isken's avatar
wip    
Marius Isken committed
326
327
328
        self.render('targets.html',
                    contribution_plot=autoload_server(
                        None,
Marius Isken's avatar
Marius Isken committed
329
330
                        app_path='/contributions',
                        url='plots'),
Marius Isken's avatar
wip    
Marius Isken committed
331
332
                    pages=pages,
                    problem=self.config.problem)
marius's avatar
marius committed
333
334
335


pages = OrderedDict([
Marius Isken's avatar
Marius Isken committed
336
    ('Rundirs', Rundirs),
Marius Isken's avatar
wip    
Marius Isken committed
337
    ('Summary', Misfit),
Marius Isken's avatar
wip    
Marius Isken committed
338
    ('Misfit', Misfit),
Marius Isken's avatar
Marius Isken committed
339
    ('Parameters', Parameters),
Marius Isken's avatar
Marius Isken committed
340
    # ('Targets', Targets),
marius's avatar
marius committed
341
342
343
])


344
class BaraddurConfig(Object):
Marius Isken's avatar
Marius Isken committed
345
346
347
348
    rundir = String.T()
    project_dir = String.T(
        help='Grond project dir.',
        optional=True)
349
    debug = Bool.T(
350
        default=False,
351
        optional=True)
Marius Isken's avatar
wip    
Marius Isken committed
352
353
354
355
    hosts = List.T(
        String.T(),
        default=['*'],
        optional=True,
marius's avatar
wip    
marius committed
356
        help='List of allowed hosts, default is all \'*\'.')
marius's avatar
wip    
marius committed
357
358
    port = Int.T(
        default=8080,
Marius Isken's avatar
wip    
Marius Isken committed
359
360
        optional=True,
        help='Port to listen on.')
361

Marius Isken's avatar
Marius Isken committed
362
363
364
365
366
    def __init__(self, *args, **kwargs):
        Object.__init__(self, *args, **kwargs)
        self._rundir = None
        self._model = None

367
368
369
370
    @property
    def problem(self):
        return grond.core.load_problem_info(self.rundir)

Marius Isken's avatar
Marius Isken committed
371
372
373
374
375
376
377
378
379
380
381
382
383
384
    @property
    def rundir(self):
        return self._rundir

    @rundir.getter
    def rundir(self):
        if self._rundir is None:
            if len(self.available_rundirs) > 0:
                self.rundir = self.available_rundirs[0]
        return self._rundir

    @rundir.setter
    def rundir(self, value):
        self._rundir = value
Marius Isken's avatar
wip    
Marius Isken committed
385
        self._model = BaraddurModel(rundir=self._rundir)
Marius Isken's avatar
Marius Isken committed
386
387
388
389

    @property
    def model(self):
        if self._model is None:
Marius Isken's avatar
wip    
Marius Isken committed
390
            self._model = BaraddurModel(rundir=self.rundir)
Marius Isken's avatar
Marius Isken committed
391
392
393
394
395
396
397
        return self._model

    @property
    def available_rundirs(self):
        rundirs = []
        for d in [d for d in glob.glob(op.join(self.project_dir, '*'))
                  if op.isdir(d)]:
Marius Isken's avatar
wip    
Marius Isken committed
398
            if BaraddurModel.validate_rundir(d):
Marius Isken's avatar
Marius Isken committed
399
400
401
                rundirs.append(d)
        return rundirs

402
403

class Baraddur(BokehServer):
Marius Isken's avatar
Marius Isken committed
404
405
406
407
408
409
410
411

    def __init__(self, config=None, project_dir=None, *args, **kwargs):
        if config is not None:
            self.config = config
        elif project_dir is not None:
            self.config = BaraddurConfig(project_dir=project_dir)
        else:
            raise AttributeError('No project dir set!')
412
        self.ioloop = tornado.ioloop.IOLoop.current()
Marius Isken's avatar
wip    
Marius Isken committed
413
        port_offset = 0
414

Marius Isken's avatar
wip    
Marius Isken committed
415
416
417
418
419
        while True:
            try:
                BokehServer.__init__(
                    self,
                    self.get_bokeh_apps(),
420
                    io_loop=self.ioloop,
Marius Isken's avatar
wip    
Marius Isken committed
421
422
423
424
425
426
427
                    extra_patterns=self.get_tornado_handlers(),
                    port=self.config.port + port_offset,
                    host=self.config.hosts)
                break
            except socket.error as se:
                if se.errno == 98 and port_offset < 50:  # Port in use
                    port_offset += 1
Marius Isken's avatar
Marius Isken committed
428
429
430
                    logger.warning('Port %d in use, bailing to %d'
                                   % (self.config.port + port_offset - 1,
                                      self.config.port + port_offset))
Marius Isken's avatar
wip    
Marius Isken committed
431
432
                else:
                    raise se
433
        tornado_app = self._tornado
434
435
        tornado_app.settings['template_path'] = op.join(
            op.dirname(op.abspath(__file__)), 'templates')
436
437
438
439
440
441
442
443
444
445

        if self.config.debug:
            tornado_app.settings.setdefault('autoreload', True)
            tornado_app.settings.setdefault('compiled_template_cache', False)
            tornado_app.settings.setdefault('static_hash_cache', False)
            tornado_app.settings.setdefault('serve_traceback', True)
            # Automatically reload modified modules
            from tornado import autoreload
            autoreload.start()

Marius Isken's avatar
wip    
Marius Isken committed
446
            logging.getLogger('').setLevel(logging.DEBUG)
447
448
449
450
451
452
453
454
455
456

    def get_bokeh_apps(self):
        bokeh_apps = {}
        for tornado_handler in pages.itervalues():
            handler_docs = getattr(tornado_handler, 'bokeh_handlers',
                                   None)
            if handler_docs is None:
                continue

            for url, bokeh_handler in handler_docs.iteritems():
Marius Isken's avatar
Marius Isken committed
457
                bokeh_apps['/plots/%s' % url] = Application(bokeh_handler(
458
459
460
461
                    self.config))
        return bokeh_apps

    def get_tornado_handlers(self):
Marius Isken's avatar
Marius Isken committed
462
463
        return [(r'/', pages.values()[0],
                 {'config': self.config})] +\
464
465
               [(r'/%s' % title, handler,
                 {'config': self.config})
Marius Isken's avatar
Marius Isken committed
466
                for title, handler in pages.iteritems()] +\
Marius Isken's avatar
Marius Isken committed
467
468
               [(r'/res/(.*)', StaticFileHandler,
                {'path': op.join(op.dirname(__file__), 'res')})]
Marius Isken's avatar
Marius Isken committed
469

470
    def start(self, signal=None):
Marius Isken's avatar
Marius Isken committed
471
472
        logger.info('Starting Baraddur server on http://%s:%d'
                    % ('localhost', self.port))
473
474
475
476
477
478
479

        if signal is not None:
            def shutdown():
                if not signal.empty():
                    self.stop()
            tornado.ioloop.PeriodicCallback(shutdown, 2000).start()

480
        BokehServer.start(self)
481
482
483
484
485
486
        self.ioloop.start()

    def stop(self, *args, **kwargs):
        logger.info('Stopping Baraddur server...')
        BokehServer.stop(self)
        self.ioloop.stop()
487
488


Marius Isken's avatar
Marius Isken committed
489
if __name__ == '__main__':
490
491
492
    baraddur = Baraddur(
        rundir='/home/marius/Development/testing/grond/rundir')
    baraddur.start()