server.py 14.9 KB
Newer Older
marius's avatar
marius committed
1
2
import tornado.ioloop
import grond
marius's avatar
wip    
marius committed
3
import os
Marius Isken's avatar
Marius Isken committed
4
import glob
marius's avatar
marius committed
5
import os.path as op
Marius Isken's avatar
Marius Isken committed
6
import logging
7
import numpy as num
Marius Isken's avatar
wip    
Marius Isken committed
8
import socket
marius's avatar
marius committed
9
10

from collections import OrderedDict
Marius Isken's avatar
Marius Isken committed
11
from datetime import date
marius's avatar
marius committed
12

Marius Isken's avatar
wip    
Marius Isken committed
13
from pyrocko.guts import Object, Bool, String, Int, List
14

Marius Isken's avatar
Marius Isken committed
15
from tornado.web import RequestHandler, StaticFileHandler
marius's avatar
marius committed
16
17
18
from tornado import gen

from bokeh.embed import autoload_server
Marius Isken's avatar
Marius Isken committed
19
20
21
from bokeh.application import Application
from bokeh.server.server import Server as BokehServer
from bokeh.application.handlers import Handler as BokehHandler
marius's avatar
marius committed
22

Marius Isken's avatar
Marius Isken committed
23
24
from bokeh.models import ColumnDataSource
from bokeh import layouts
marius's avatar
marius committed
25
26
from bokeh.plotting import figure

Marius Isken's avatar
wip    
Marius Isken committed
27
28
29
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('grond.baraddur')

marius's avatar
marius committed
30

Marius Isken's avatar
Marius Isken committed
31
32
33
34
35
36
37
38
39
def makeColorGradient(misfits, fr=1., fg=.5, fb=1.,
                      pr=0, pg=2.5, pb=4):
    misfits /= misfits.max()
    r = num.sin(fr * misfits + pr) * 127 + 128
    g = num.sin(fg * misfits + pg) * 127 + 128
    b = num.sin(fb * misfits + pb) * 127 + 128
    return ['#%02x%02x%02x' % (r[i], g[i], b[i]) for i in xrange(misfits.size)]


40
41
42
43
class BaraddurRequestHandler(RequestHandler):
    def initialize(self, config):
        self.config = config

Marius Isken's avatar
Marius Isken committed
44

45
46
47
48
49
50
class BaraddurBokehHandler(BokehHandler):
    def __init__(self, config, *args, **kwargs):
        BokehHandler.__init__(self, *args, **kwargs)
        self.config = config


marius's avatar
wip    
marius committed
51
class GrondBokehModel(object):
Marius Isken's avatar
Marius Isken committed
52
53
    def __init__(self, rundir):
        self.set_rundir(rundir)
marius's avatar
wip    
marius committed
54
55
56

    def set_rundir(self, rundir):
        logger.debug('Loading problem from %s' % rundir)
Marius Isken's avatar
Marius Isken committed
57
        self.rundir = op.abspath(rundir)
marius's avatar
wip    
marius committed
58
59
60
61
62
        self.problem = grond.core.load_problem_info(self.rundir)
        self.parameters = self.problem.parameters
        self.nparameters = self.problem.nparameters
        self.ntargets = self.problem.ntargets

Marius Isken's avatar
Marius Isken committed
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
    def _jp(self, file):
        return op.join(self.rundir, file)

    @property
    def start_time(self):
        stat = os.stat(self._jp('problem.yaml'))
        try:
            t = stat.st_birthtime
        except AttributeError:
            t = stat.st_mtime
        return date.fromtimestamp(t).strftime('%Y-%m-%d %H:%M')

    @property
    def name(self):
        return op.split(self.rundir)[-1]

    @property
    def best_misfit(self):
        _, data = self.get_misfits(keys='target_mean')
        return data['target_mean'].min()

    @property
    def niterations(self):
        return op.getsize(self._jp('models')) / (self.nparameters * 8)

    @staticmethod
    def validate_rundir(rundir):

        def jp(fn):
            return op.join(rundir, fn)

        for fn in ['models', 'misfits', 'problem.yaml']:
            if op.lexists(jp(fn)):
                if op.getsize(jp(fn)) == 0:
                    return False
            else:
                return False
        return True

    def get_models(self, skip_nmodels=0, keys=None):
        fn = self._jp('models')

marius's avatar
wip    
marius committed
105
106
107
108
109
110
        with open(fn, 'r') as f:
            nmodels = os.fstat(f.fileno()).st_size / (self.nparameters * 8)
            nmodels -= skip_nmodels
            f.seek(skip_nmodels * self.nparameters * 8)
            data = num.fromfile(
                f, dtype='<f8', count=nmodels * self.nparameters)\
Marius Isken's avatar
Marius Isken committed
111
                .astype(num.float32)
marius's avatar
wip    
marius committed
112
113
114
115
116
117
118

        nmodels = data.size/self.nparameters - skip_nmodels
        models = data.reshape((nmodels, self.nparameters))

        mods_dict = {}
        for ip, par in enumerate(self.parameters):
            mods_dict[par.name] = models[:, ip]
Marius Isken's avatar
Marius Isken committed
119
120
121
122
123
124
125
126
        mods_dict['niter'] = num.arange(nmodels, dtype=num.int)\
            + skip_nmodels + 1

        if keys is not None:
            for k in mods_dict.keys():
                if k not in keys:
                    mods_dict.pop(k)

marius's avatar
wip    
marius committed
127
128
        return nmodels, mods_dict

Marius Isken's avatar
Marius Isken committed
129
130
    def get_misfits(self, skip_nmodels=0, keys=None):
        fn = self._jp('misfits')
marius's avatar
wip    
marius committed
131
132
133
134
135
136
137

        with open(fn, 'r') as f:
            nmodels = os.fstat(f.fileno()).st_size / (self.nparameters * 8)
            nmodels -= skip_nmodels
            f.seek(skip_nmodels * self.ntargets * 2 * 8)
            data = num.fromfile(
                f, dtype='<f8', count=nmodels*self.ntargets*2)\
Marius Isken's avatar
Marius Isken committed
138
                .astype(num.float32)
marius's avatar
wip    
marius committed
139
140
141
142
143
144
145
146
147
148

        data = data.reshape((nmodels, self.ntargets*2))
        combi = num.empty_like(data)
        combi[:, 0::2] = data[:, :self.ntargets]
        combi[:, 1::2] = data[:, self.ntargets:]
        misfits = combi.reshape((nmodels, self.ntargets, 2))

        mf_dict = {}
        for it in xrange(self.ntargets):
            mf_dict['target_%03d' % it] = misfits[:, it, 0]
Marius Isken's avatar
Marius Isken committed
149
        mf_dict['target_mean'] = num.mean(misfits, axis=1)[:, 0]
marius's avatar
wip    
marius committed
150

Marius Isken's avatar
Marius Isken committed
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
        mf_dict['niter'] = num.arange(nmodels, dtype=num.int)\
            + skip_nmodels + 1

        if keys is not None:
            for k in mf_dict.keys():
                if k not in keys:
                    mf_dict.pop(k)

        return nmodels, mf_dict


class Rundirs(BaraddurRequestHandler):

    def get(self):
        models = [GrondBokehModel(rundir=rd)
                  for rd in self.config.available_rundirs]
        self.render('rundirs.html',
                    pages=pages,
                    project_dir=self.config.project_dir,
                    models=models,
                    active_rundir=self.config.rundir)

    def post(self):
        rundir = self.get_argument('rundir', None)
        if rundir is not None:
            self.config.rundir = rundir
        self.get()


class Summary(BaraddurRequestHandler):

    @gen.coroutine
    def get(self):
        self.render('summary.html',
                    pages=pages,
                    problem=self.config.problem)
marius's avatar
wip    
marius committed
187
188


189
190
191
class Status(BaraddurRequestHandler):

    class MisfitsPlot(BaraddurBokehHandler):
Marius Isken's avatar
Marius Isken committed
192
193

        def modify_document(self, doc):
194
            self.nmodels = 0
Marius Isken's avatar
Marius Isken committed
195
196
197
198
199
            self.source = ColumnDataSource()
            self.source.add(num.ndarray(0, dtype=num.float32),
                            'target_mean')
            self.source.add(num.ndarray(0, dtype=num.float32),
                            'niter')
Marius Isken's avatar
Marius Isken committed
200
201
202
203
204
            self.update_misfits()

            plot = figure(webgl=True,
                          x_axis_label='Iteration #',
                          y_axis_label='Misfit')
Marius Isken's avatar
Marius Isken committed
205
            plot.scatter('niter', 'target_mean',
Marius Isken's avatar
Marius Isken committed
206
207
208
209
210
211
212
                         source=self.source, alpha=.4)

            doc.add_root(plot)
            doc.add_periodic_callback(self.update_misfits, 1e3)

        @gen.coroutine
        def update_misfits(self):
Marius Isken's avatar
Marius Isken committed
213
214
215
216
            new_nmodels, new_data = self.config.model.get_misfits(
                skip_nmodels=self.nmodels,
                keys=self.source.data.keys())
            self.source.stream(new_data)
217
            self.nmodels += new_nmodels
Marius Isken's avatar
Marius Isken committed
218

Marius Isken's avatar
Marius Isken committed
219
    bokeh_handlers = {'misfit': MisfitsPlot}
Marius Isken's avatar
Marius Isken committed
220
221
222
223
224

    @gen.coroutine
    def get(self):
        self.render('status.html',
                    pages=pages,
Marius Isken's avatar
Marius Isken committed
225
226
227
228
                    misfit_plot=autoload_server(
                        None,
                        app_path='/misfit',
                        url='plots'),
229
                    problem=self.config.problem)
Marius Isken's avatar
Marius Isken committed
230
231


232
class Parameters(BaraddurRequestHandler):
Marius Isken's avatar
Marius Isken committed
233

234
    class ParameterPlots(BaraddurBokehHandler):
Marius Isken's avatar
Marius Isken committed
235
236
237
238

        ncols = 4

        def modify_document(self, doc):
239
            self.nmodels = 0
240
241
            problem = self.config.problem

242
            self.source = ColumnDataSource()
Marius Isken's avatar
Marius Isken committed
243
244
245
            for p in ['niter'] + [p.name for p in problem.parameters]:
                self.source.add(num.ndarray(0, dtype=num.float32), p)
            self.model = GrondBokehModel(self.config.rundir)
Marius Isken's avatar
Marius Isken committed
246
247
248
249
250
251
252
            self.update_parameters()

            plots = []
            for par in problem.parameters:
                fig = figure(webgl=True,
                             x_axis_label='Iteration #',
                             y_axis_label='%s [%s]' % (par.label, par.unit))
Marius Isken's avatar
Marius Isken committed
253
                fig.scatter('niter', par.name,
254
                            source=self.source, alpha=.4)
Marius Isken's avatar
Marius Isken committed
255
                plots.append(fig)
Marius Isken's avatar
wip    
Marius Isken committed
256
            plots += [None] * (self.ncols - (len(plots) % self.ncols))
Marius Isken's avatar
Marius Isken committed
257
258
259
260
261
262
263

            grid = layouts.gridplot(
                plots,
                responsive=True,
                ncols=self.ncols)

            doc.add_root(grid)
264
            doc.add_periodic_callback(self.update_parameters, 2.5*1e3)
Marius Isken's avatar
Marius Isken committed
265
266
267

        @gen.coroutine
        def update_parameters(self):
Marius Isken's avatar
Marius Isken committed
268
269
270
            new_nmodels, new_data = self.config.model.get_models(
                skip_nmodels=self.nmodels,
                keys=self.source.data.keys())
271
272
            self.source.stream(new_data)
            self.nmodels += new_nmodels
Marius Isken's avatar
Marius Isken committed
273

Marius Isken's avatar
Marius Isken committed
274
    bokeh_handlers = {'parameters': ParameterPlots}
marius's avatar
marius committed
275
276
277
278

    @gen.coroutine
    def get(self):

Marius Isken's avatar
Marius Isken committed
279
        self.render('parameters.html',
Marius Isken's avatar
Marius Isken committed
280
281
282
                    pages=pages,
                    parameter_plot=autoload_server(
                        None,
Marius Isken's avatar
Marius Isken committed
283
284
                        app_path='/parameters',
                        url='plots'),
285
                    problem=self.config.problem)
marius's avatar
marius committed
286
287


Marius Isken's avatar
wip    
Marius Isken committed
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
class Targets(BaraddurRequestHandler):

    class TargetContributionPlot(BaraddurBokehHandler):

        def modify_document(self, doc):
            self.nmodels = 0
            self.source = ColumnDataSource()
            self.update_contributions()

            plot = figure(webgl=True,
                          x_axis_label='Iteration #',
                          y_axis_label='Misfit')

            doc.add_root(plot)
            doc.add_periodic_callback(self.update_contributions, 1e3)

        @gen.coroutine
        def update_contributions(self):
            mx, misfits = grond.core.load_problem_data(
                self.config.rundir, self.config.problem,
                skip_models=self.nmodels)

            new_nmodels = mx.shape[0]
            self.nmodels += new_nmodels

Marius Isken's avatar
Marius Isken committed
313
    bokeh_handlers = {'contributions': TargetContributionPlot}
Marius Isken's avatar
Marius Isken committed
314

marius's avatar
marius committed
315
316
    @gen.coroutine
    def get(self):
Marius Isken's avatar
wip    
Marius Isken committed
317
318
319
        self.render('targets.html',
                    contribution_plot=autoload_server(
                        None,
Marius Isken's avatar
Marius Isken committed
320
321
                        app_path='/contributions',
                        url='plots'),
Marius Isken's avatar
wip    
Marius Isken committed
322
323
                    pages=pages,
                    problem=self.config.problem)
marius's avatar
marius committed
324
325
326


pages = OrderedDict([
Marius Isken's avatar
Marius Isken committed
327
    ('Rundirs', Rundirs),
Marius Isken's avatar
wip    
Marius Isken committed
328
    ('Summary', Summary),
Marius Isken's avatar
Marius Isken committed
329
330
    ('Status', Status),
    ('Parameters', Parameters),
Marius Isken's avatar
Marius Isken committed
331
    # ('Targets', Targets),
marius's avatar
marius committed
332
333
334
])


335
class BaraddurConfig(Object):
Marius Isken's avatar
Marius Isken committed
336
337
338
339
    rundir = String.T()
    project_dir = String.T(
        help='Grond project dir.',
        optional=True)
340
    debug = Bool.T(
341
        default=False,
342
        optional=True)
Marius Isken's avatar
wip    
Marius Isken committed
343
344
345
346
    hosts = List.T(
        String.T(),
        default=['*'],
        optional=True,
marius's avatar
wip    
marius committed
347
        help='List of allowed hosts, default is all \'*\'.')
marius's avatar
wip    
marius committed
348
349
    port = Int.T(
        default=8080,
Marius Isken's avatar
wip    
Marius Isken committed
350
351
        optional=True,
        help='Port to listen on.')
352

Marius Isken's avatar
Marius Isken committed
353
354
355
356
357
    def __init__(self, *args, **kwargs):
        Object.__init__(self, *args, **kwargs)
        self._rundir = None
        self._model = None

358
359
360
361
    @property
    def problem(self):
        return grond.core.load_problem_info(self.rundir)

Marius Isken's avatar
Marius Isken committed
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
    @property
    def rundir(self):
        return self._rundir

    @rundir.getter
    def rundir(self):
        if self._rundir is None:
            if len(self.available_rundirs) > 0:
                self.rundir = self.available_rundirs[0]
        return self._rundir

    @rundir.setter
    def rundir(self, value):
        self._rundir = value
        self._model = GrondBokehModel(rundir=self._rundir)

    @property
    def model(self):
        if self._model is None:
            self._model = GrondBokehModel(rundir=self.rundir)
        return self._model

    @property
    def available_rundirs(self):
        rundirs = []
        for d in [d for d in glob.glob(op.join(self.project_dir, '*'))
                  if op.isdir(d)]:
            if GrondBokehModel.validate_rundir(d):
                rundirs.append(d)
        return rundirs

393
394

class Baraddur(BokehServer):
Marius Isken's avatar
Marius Isken committed
395
396
397
398
399
400
401
402

    def __init__(self, config=None, project_dir=None, *args, **kwargs):
        if config is not None:
            self.config = config
        elif project_dir is not None:
            self.config = BaraddurConfig(project_dir=project_dir)
        else:
            raise AttributeError('No project dir set!')
403
        self.ioloop = tornado.ioloop.IOLoop.current()
Marius Isken's avatar
wip    
Marius Isken committed
404
        port_offset = 0
405

Marius Isken's avatar
wip    
Marius Isken committed
406
407
408
409
410
        while True:
            try:
                BokehServer.__init__(
                    self,
                    self.get_bokeh_apps(),
411
                    io_loop=self.ioloop,
Marius Isken's avatar
wip    
Marius Isken committed
412
413
414
415
416
417
418
                    extra_patterns=self.get_tornado_handlers(),
                    port=self.config.port + port_offset,
                    host=self.config.hosts)
                break
            except socket.error as se:
                if se.errno == 98 and port_offset < 50:  # Port in use
                    port_offset += 1
Marius Isken's avatar
Marius Isken committed
419
420
421
                    logger.warning('Port %d in use, bailing to %d'
                                   % (self.config.port + port_offset - 1,
                                      self.config.port + port_offset))
Marius Isken's avatar
wip    
Marius Isken committed
422
423
                else:
                    raise se
424
        tornado_app = self._tornado
425
426
        tornado_app.settings['template_path'] = op.join(
            op.dirname(op.abspath(__file__)), 'templates')
427
428
429
430
431
432
433
434
435
436

        if self.config.debug:
            tornado_app.settings.setdefault('autoreload', True)
            tornado_app.settings.setdefault('compiled_template_cache', False)
            tornado_app.settings.setdefault('static_hash_cache', False)
            tornado_app.settings.setdefault('serve_traceback', True)
            # Automatically reload modified modules
            from tornado import autoreload
            autoreload.start()

Marius Isken's avatar
wip    
Marius Isken committed
437
            logging.getLogger('').setLevel(logging.DEBUG)
438
439
440
441
442
443
444
445
446
447

    def get_bokeh_apps(self):
        bokeh_apps = {}
        for tornado_handler in pages.itervalues():
            handler_docs = getattr(tornado_handler, 'bokeh_handlers',
                                   None)
            if handler_docs is None:
                continue

            for url, bokeh_handler in handler_docs.iteritems():
Marius Isken's avatar
Marius Isken committed
448
                bokeh_apps['/plots/%s' % url] = Application(bokeh_handler(
449
450
451
452
                    self.config))
        return bokeh_apps

    def get_tornado_handlers(self):
Marius Isken's avatar
Marius Isken committed
453
454
        return [(r'/', pages.values()[0],
                 {'config': self.config})] +\
455
456
               [(r'/%s' % title, handler,
                 {'config': self.config})
Marius Isken's avatar
Marius Isken committed
457
                for title, handler in pages.iteritems()] +\
Marius Isken's avatar
Marius Isken committed
458
459
               [(r'/res/(.*)', StaticFileHandler,
                {'path': op.join(op.dirname(__file__), 'res')})]
Marius Isken's avatar
Marius Isken committed
460

461
    def start(self, signal=None):
Marius Isken's avatar
Marius Isken committed
462
463
        logger.info('Starting Baraddur server on http://%s:%d'
                    % ('localhost', self.port))
464
465
466
467
468
469
470

        if signal is not None:
            def shutdown():
                if not signal.empty():
                    self.stop()
            tornado.ioloop.PeriodicCallback(shutdown, 2000).start()

471
        BokehServer.start(self)
472
473
474
475
476
477
        self.ioloop.start()

    def stop(self, *args, **kwargs):
        logger.info('Stopping Baraddur server...')
        BokehServer.stop(self)
        self.ioloop.stop()
478
479


Marius Isken's avatar
Marius Isken committed
480
if __name__ == '__main__':
481
482
483
    baraddur = Baraddur(
        rundir='/home/marius/Development/testing/grond/rundir')
    baraddur.start()