Merge branch 'master' of https://gitext.gfz-potsdam.de/heimann/lassie

README.md

@@ -189,3 +189,18 @@ Now run the detector with
 lassie scan config.yaml
 ```
 
+#### Scrutinize detections
+
+After the detection has finished have a look at the results using Pyrocko's
+[Snuffler](http://emolch.github.io/pyrocko/current/snuffler.html):
+
+```bash
+lassie snuffle config.yaml
+```
+
+![example snuffling](doc/gx/snuffling_example.png "Test with synthetic events")
+
+Snuffler opens loading waveform data together with station meta data and
+detections as event markers. Load a reference catalog to compare the detections
+to and scrutinize detection performance at different detection thresholds,
+interactively.

apps/lassie

@@ -27,14 +27,16 @@ def str_to_time(s):
 
 subcommand_descriptions = {
     'init': 'create initial configuration file',
-    'scan': 'detect seismic events',
-    'map-geometry': 'make station map'
+    'search': 'detect seismic events',
+    'map-geometry': 'make station map',
+    'snuffle': 'snuffle'
 }
 
 subcommand_usages = {
     'init': 'init',
-    'scan': 'scan <configfile> [options]',
-    'map-geometry': 'map-geometry <configfile> [options]',
+    'search': 'search <configfile> [options]',
+    'map-geometry': 'map-geometry <configfile> [options] <output.(png|pdf)',
+    'snuffle': 'snuffle <configfile>',
 }
 
 subcommands = subcommand_descriptions.keys()
@@ -49,8 +51,9 @@ usage = program_name + ''' <subcommand> [options] [--] <arguments> ...
 Subcommands:
 
     init          %(init)s
-    scan          %(scan)s
+    search        %(search)s
     map-geometry  %(map_geometry)s
+    snuffle       %(snuffle)s
 
 To get further help and a list of available options for any subcommand run:
 
@@ -67,9 +70,9 @@ def add_common_options(parser):
         type='choice',
         choices=('critical', 'error', 'warning', 'info', 'debug'),
         default='info',
-        help ='set logger level to '
-              '"critical", "error", "warning", "info", or "debug". '
-              'Default is "%default".')
+        help='set logger level to '
+             '"critical", "error", "warning", "info", or "debug". '
+             'Default is "%default".')
 
 
 def process_common_options(options):
@@ -169,6 +172,10 @@ stations_path: '%(stations_path)s'
 data_paths:
 %(s_data_paths)s
 
+## name template for Lassie's output directory. The placeholder
+## "${config_name}" will be replaced with the basename of the config file.
+run_path: '${config_name}.turd'
+
 ## Processing time interval (default: use time interval of available data)
 # tmin: '2012-02-06 04:20:00'
 # tmax: '2012-02-06 04:30:00'
@@ -193,39 +200,90 @@ autogrid_radius_factor: 1.5
 ## Grid density factor used when automatically choosing a grid
 autogrid_density_factor: 10.0
 
-## Image function composition
+## Composition of image function
 image_function_contributions:
-- !lassie.WavePacketIFC
+
+- !lassie.OnsetIFC
   name: 'P'
-  weight: 1.0
+  weight: 30.0
   fmin: 1.0
   fmax: 15.0
-  fsmooth_factor: 0.1
-  shifter: !lassie.VelocityShifter
-    velocity: 3500.
+  short_window: 1.0
+  window_ratio: 8.0
+  fsmooth: 0.2
+  fnormalize: 0.02
+  #shifter: !lassie.VelocityShifter
+  #  velocity: 6000.
+  shifter: !lassie.CakePhaseShifter
+    timing: '{stored:p}'
+    earthmodel_id: 'swiss'
+
 - !lassie.WavePacketIFC
   name: 'S'
   weight: 1.0
   fmin: 1.0
-  fmax: 10.0
-  fsmooth_factor: 0.1
-  shifter: !lassie.VelocityShifter
-    velocity: 2500.
+  fmax: 8.0
+  fsmooth: 0.05
+  #shifter: !lassie.VelocityShifter
+  #  velocity: 3300.
+  shifter: !lassie.CakePhaseShifter
+    # factor: 1.0
+    # offset: 1.0
+    timing: '{stored:s}'
+    earthmodel_id: 'swiss'
 
 ## Whether to divide image function frames by their mean value
-sharpness_normalization: true
+sharpness_normalization: false
 
 ## Threshold on detector function
-detector_threshold: 100.0
-
-## Output filename for detections
-detections_path: 'detections.txt'
+detector_threshold: 150.
+
+## Whether to create a figure for every detection and save it in the output
+## directory
+save_figures: true
+
+## Mapping of phase ID to phase definition in cake syntax (used e.g. in the 
+## CakePhaseShifter config sections)
+tabulated_phases:
+- !pf.TPDef
+  id: 'p'
+  definition: 'P,p'
+- !pf.TPDef
+  id: 's'
+  definition: 'S,s'
+
+## Mapping of earthmodel ID  to the actual earth model in nd format (used in 
+## the CakePhaseShifter config sections)
+earthmodels:
+- !lassie.CakeEarthmodel
+  id: 'swiss'
+  earthmodel_1d: |2
+    0.0 5.53 3.10  2.75
+    2.0 5.53 3.10  2.75
+    2.0 5.80 3.25  2.75
+    5.0 5.80 3.25  2.75
+    5.0 5.83 3.27  2.75
+    8.0 5.83 3.27  2.75
+    8.0 5.95 3.34  2.8
+    13.0 5.95 3.34  2.8
+    13.0 5.96 3.34  2.8
+    22.0 5.96 3.34  2.8
+    22.0 6.53 3.66  2.8
+    30.0 6.53 3.66  2.8
+    30.0 7.18 4.03 3.3
+    40.0 7.18 4.03 3.3
+    40.0 7.53 4.23 3.3
+    50.0 7.53 4.23 3.3
+    50.0 7.83 4.39 3.3
+    60.0 7.83 4.39 3.3
+    60.0 8.15 4.57 3.3
+    120.0 8.15 4.57 3.3
 ''' % dict(
         stations_path=stations_path,
         s_data_paths=s_data_paths)
 
 
-def command_scan(args):
+def command_search(args):
     def setup(parser):
         parser.add_option(
             '--force', dest='force', action='store_true',
@@ -239,6 +297,11 @@ def command_scan(args):
             '--show-movie', dest='show_movie', action='store_true',
             help='show movie when showing detections')
 
+        parser.add_option(
+            '--show-window-traces', dest='show_window_traces',
+            action='store_true',
+            help='show preprocessed traces for every processing time window')
+
         parser.add_option(
             '--stop-after-first', dest='stop_after_first', action='store_true',
             help='show plot for every detection found')
@@ -253,7 +316,15 @@ def command_scan(args):
             help='end of processing time window '
                  '(overrides config file settings)')
 
-    parser, options, args = cl_parse('scan', args, setup=setup)
+        parser.add_option(
+            '--nworkers', dest='nworkers', metavar="N",
+            help='use N cpus in parallel')
+
+        parser.add_option(
+            '--speak', dest='bark', action='store_true',
+            help='alert on detection of events')
+
+    parser, options, args = cl_parse('search', args, setup=setup)
     if len(args) != 1:
         help_and_die(parser, 'missing argument')
 
@@ -268,14 +339,22 @@ def command_scan(args):
         if options.tmax:
             tmax = str_to_time(options.tmax)
 
-        lassie.scan(
+        if options.nworkers:
+            nparallel = int(options.nworkers)
+        else:
+            nparallel = None
+
+        lassie.search(
             config,
             override_tmin=tmin,
             override_tmax=tmax,
             force=options.force,
             show_detections=options.show_detections,
             show_movie=options.show_movie,
-            stop_after_first=options.stop_after_first)
+            show_window_traces=options.show_window_traces,
+            stop_after_first=options.stop_after_first,
+            nparallel=nparallel,
+            bark=options.bark)
 
     except lassie.LassieError, e:
         die(str(e))
@@ -292,6 +371,17 @@ def command_map_geometry(args):
     lassie.map_geometry(config, output_path)
 
 
+def command_snuffle(args):
+    parser, options, args = cl_parse('snuffle', args)
+    if len(args) != 1:
+        help_and_die(parser, 'missing arguments')
+
+    config_path = args[0]
+    config = lassie.read_config(config_path)
+
+    lassie.snuffle(config)
+
+
 if __name__ == '__main__':
 
     usage_sub = 'fomosto %s [options]'

setup.py

@@ -7,4 +7,4 @@ setup(
     packages=['lassie'],
     package_dir={'lassie': 'src'},
     scripts=['apps/lassie'],
-    package_data={'lassie': []})
+    package_data={'lassie': ['data/*.wav']})

src/__init__.py

@@ -6,6 +6,7 @@ from lassie.ifc import *  # noqa
 from lassie.core import *  # noqa
 from lassie.plot import *  # noqa
 from lassie.config import *  # noqa
+from lassie.snuffling  import *  # noqa
 
 
 __version__ = '0.2'

src/common.py

+import os.path as op
+from string import Template
 from collections import defaultdict
 
+from pyrocko.guts import Object, String
+from pyrocko.gf import Earthmodel1D
+
 guts_prefix = 'lassie'
 
 
+def data_file(fn):
+    return op.join(op.split(__file__)[0], 'data', fn)
+
+
 class LassieError(Exception):
     pass
 
 
+class Earthmodel(Object):
+    id = String.T()
+
+
+class CakeEarthmodel(Earthmodel):
+    earthmodel_1d = Earthmodel1D.T()
+
+
 def grouped_by(l, key):
     d = defaultdict(list)
     for x in l:
@@ -14,6 +31,103 @@ def grouped_by(l, key):
 
     return d
 
+
+def xjoin(basepath, path):
+    if path is None and basepath is not None:
+        return basepath
+    elif op.isabs(path) or basepath is None:
+        return path
+    else:
+        return op.join(basepath, path)
+
+
+def xrelpath(path, start):
+    if op.isabs(path):
+        return path
+    else:
+        return op.relpath(path, start)
+
+
+class Path(String):
+    pass
+
+
+class HasPaths(Object):
+    path_prefix = Path.T(optional=True)
+
+    def __init__(self, *args, **kwargs):
+        Object.__init__(self, *args, **kwargs)
+        self._basepath = None
+        self._parent_path_prefix = None
+
+    def set_basepath(self, basepath, parent_path_prefix=None):
+        self._basepath = basepath
+        self._parent_path_prefix = parent_path_prefix
+        for (prop, val) in self.T.ipropvals(self):
+            if isinstance(val, HasPaths):
+                val.set_basepath(
+                    basepath, self.path_prefix or self._parent_path_prefix)
+
+    def get_basepath(self):
+        assert self._basepath is not None
+        return self._basepath
+
+    def change_basepath(self, new_basepath, parent_path_prefix=None):
+        assert self._basepath is not None
+
+        self._parent_path_prefix = parent_path_prefix
+        if self.path_prefix or not self._parent_path_prefix:
+
+            self.path_prefix = op.normpath(xjoin(xrelpath(
+                self._basepath, new_basepath), self.path_prefix))
+
+        for val in self.T.ivals(self):
+            if isinstance(val, HasPaths):
+                val.change_basepath(
+                    new_basepath, self.path_prefix or self._parent_path_prefix)
+
+        self._basepath = new_basepath
+
+    def expand_path(self, path, extra=None):
+        assert self._basepath is not None
+
+        if extra is None:
+            def extra(path):
+                return path
+
+        path_prefix = self.path_prefix or self._parent_path_prefix
+
+        if path is None:
+            return None
+        elif isinstance(path, basestring):
+            return extra(
+                op.normpath(xjoin(self._basepath, xjoin(path_prefix, path))))
+        else:
+            return [
+                extra(
+                    op.normpath(xjoin(self._basepath, xjoin(path_prefix, p))))
+                for p in path]
+
+
+def expand_template(template, d):
+    try:
+        return Template(template).substitute(d)
+    except KeyError as e:
+        raise LassieError(
+            'invalid placeholder "%s" in template: "%s"' % (str(e), template))
+    except ValueError:
+        raise LassieError(
+            'malformed placeholder in template: "%s"' % template)
+
+def bark():
+    import subprocess
+    subprocess.call(['aplay', data_file('bark.wav')])
+
+
 __all__ = [
     'LassieError',
+    'Earthmodel',
+    'CakeEarthmodel',
+    'HasPaths',
+    'Path',
 ]

src/config.py

 import logging
+import os.path as op
 
-from pyrocko.guts import Object, String, Float, Timestamp, List, Bool, load
-from pyrocko import model
+from pyrocko.guts import Object, String, Float, Timestamp, List, Bool
+from pyrocko import model, guts
+from pyrocko.fdsn import station as fs
+from pyrocko.gf import TPDef
 
 from lassie import receiver, ifc, grid, geo
+from lassie.common import Earthmodel, HasPaths, Path, LassieError, \
+    expand_template
 
 guts_prefix = 'lassie'
 
 logger = logging.getLogger('lassie.config')
 
 
-class Config(Object):
+class Config(HasPaths):
 
-    stations_path = String.T(
+    stations_path = Path.T(
         optional=True,
         help='stations file in Pyrocko format')
 
+    stations_stationxml_path = Path.T(
+        optional=True,
+        help='stations file in StationXML format')
+
     receivers = List.T(
         receiver.Receiver.T(),
         help='receiver coordinates if not read from file')
 
     data_paths = List.T(
-        String.T(),
+        Path.T(),
         help='list of directories paths to search for data')
 
+    events_path = Path.T(
+        optional=True,
+        help='limit processing to time windows around given events')
+
+    event_time_window_factor = Float.T(
+        default=2.,
+        help='controls length of time windows for event-wise processing')
+
     blacklist = List.T(
         String.T(),
         help='codes in the form NET.STA.LOC of receivers to be excluded')
 
+    whitelist = List.T(
+        String.T(),
+        help='codes in the form NET.STA.LOC of receivers to be included')
+
+    distance_max = Float.T(
+        optional=True,
+        help='receiver maximum distance from grid')
+
     tmin = Timestamp.T(
         optional=True,
         help='beginning of time interval to be processed')
@@ -36,9 +61,13 @@ class Config(Object):
         optional=True,
         help='end of time interval to be processed')
 
-    detections_path = String.T(
+    run_path = Path.T(
         optional=True,
-        help='output file name for for detections')
+        help='output is saved to this directory')
+
+    save_figures = Bool.T(
+        default=False,
+        help='flag to activate saving of detection figures')
 
     grid = grid.Grid.T(
         optional=True,
@@ -66,26 +95,99 @@ class Config(Object):
         default=200.,
         help='threshold on detector function')
 
+    fill_incomplete_with_zeros = Bool.T(
+        default=False,
+        help='fill incomplete trace time windows with zeros '
+             '(and let edge effects ruin your day)')
+
+    earthmodels = List.T(
+        Earthmodel.T(),
+        help='list of earthmodels usable in shifters')
+
+    tabulated_phases = List.T(
+        TPDef.T(),
+        help='list of tabulated phase definitions usable shifters')
+
+    cache_path = Path.T(
+        default='lassie_phases.cache',
+        help='directory where lassie stores tabulated phases etc.')
+
     def __init__(self, *args, **kwargs):
         Object.__init__(self, *args, **kwargs)
         self._receivers = None
         self._grid = None
+        self._events = None
+        self._config_name = 'untitled'
+
+    def set_config_name(self, config_name):
+        self._config_name = config_name
+
+    def expand_path(self, path):
+        def extra(path):
+            return expand_template(path, dict(
+                config_name=self._config_name))
+
+        return HasPaths.expand_path(self, path, extra=extra)
+
+    def get_events_path(self):
+        run_path = self.expand_path(self.run_path)
+        return op.join(run_path, 'events.list')
+
+    def get_ifm_dir_path(self):
+        run_path = self.expand_path(self.run_path)
+        return op.join(run_path, 'ifm')
+
+    def get_ifm_path_template(self):
+        return op.join(
+            self.get_ifm_dir_path(),
+            '%(station)s_%(tmin_ms)s.mseed')
+
+    def get_detections_path(self):
+        run_path = self.expand_path(self.run_path)
+        return op.join(run_path, 'detections.list')
+
+    def get_figures_path_template(self):
+        run_path = self.expand_path(self.run_path)
+        return op.join(run_path, 'figures', 'detection_%(id)06i.%(format)s')
 
     def get_receivers(self):
         '''Aggregate receivers from different sources.'''
 
+        fp = self.expand_path
+
         if self._receivers is None:
             self._receivers = list(self.receivers)
             if self.stations_path:
-                for station in model.load_stations(self.stations_path):
+                for station in model.load_stations(fp(self.stations_path)):
+                    self._receivers.append(
+                        receiver.Receiver(
+                            codes=station.nsl(),
+                            lat=station.lat,
+                            lon=station.lon,
+                            z=station.depth))
+
+            if self.stations_stationxml_path:
+                sx = fs.load_xml(filename=fp(self.stations_stationxml_path))
+                for station in sx.get_pyrocko_stations():
                     self._receivers.append(
                         receiver.Receiver(
                             codes=station.nsl(),
                             lat=station.lat,
-                            lon=station.lon))
+                            lon=station.lon,
+                            z=station.depth))
 
         return self._receivers
 
+    def get_events(self):
+        if self.events_path is None:
+            return None
+
+        if self._events is None:
+            self._events = model.load_events(self.expand_path(
+                self.events_path))
+
+        return self._events
+
     def get_grid(self):
         '''Get grid or make default grid.'''
 
@@ -104,7 +206,7 @@ class Config(Object):
                 spacing = vmin / fsmooth_max / self.autogrid_density_factor
 
                 lat0, lon0, north, east, depth = geo.bounding_box_square(
-                    *receiver.receivers_coords(receivers),
+                    *geo.points_coords(receivers),
                     scale=self.autogrid_radius_factor)
 
                 self._grid = grid.Carthesian3DGrid(
@@ -130,9 +232,24 @@ class Config(Object):
         return self._grid
 
 
-def read_config(file_path):
-    conf = load(filename=file_path)
-    return conf
+def read_config(path):
+    config = guts.load(filename=path)
+    if not isinstance(config, Config):
+        raise LassieError('invalid Lassie configuration in file "%s"' % path)
+
+    config.set_basepath(op.dirname(path) or '.')
+    config.set_config_name(op.splitext(op.basename(path))[0])
+
+    return config
+
+
+def write_config(config, path):
+    basepath = config.get_basepath()
+    dirname = op.dirname(path) or '.'
+    config.change_basepath(dirname)
+    guts.dump(config, filename=path)
+    config.change_basepath(basepath)
+
 
 __all__ = [
     'Config',

src/geo.py

 import numpy as num
 from pyrocko import orthodrome as od
+from pyrocko.guts import Object, Float
+
+
+class Point(Object):
+    lat = Float.T(default=0.0)
+    lon = Float.T(default=0.0)
+    x = Float.T(default=0.0)
+    y = Float.T(default=0.0)
+    z = Float.T(default=0.0)
+
+    @property
+    def vec(self):
+        return self.lat, self.lon, self.x, self.y, self.