Commit ccbb734c authored by Sebastian Heimann's avatar Sebastian Heimann
Browse files

more flexibility in modelling

- MisfitTarget and gf.Target are now conceptually independent
- MisfitTarget can produce more than one misfit (impl. incomplete)
parent 975d5b99
...@@ -6,7 +6,7 @@ license-file=LICENSE ...@@ -6,7 +6,7 @@ license-file=LICENSE
universal=1 universal=1
[build_ext] [build_ext]
inplace=1 inplace=0
[nosetests] [nosetests]
verbosity=2 verbosity=2
......
...@@ -19,6 +19,7 @@ from .problems.base import ProblemConfig, Problem, \ ...@@ -19,6 +19,7 @@ from .problems.base import ProblemConfig, Problem, \
from .optimizers.base import OptimizerConfig, BadProblem from .optimizers.base import OptimizerConfig, BadProblem
from .targets.base import TargetGroup from .targets.base import TargetGroup
from .targets.waveform.target import WaveformMisfitResult
from .analysers.base import AnalyserConfig from .analysers.base import AnalyserConfig
from .meta import Path, HasPaths, expand_template, GrondError, Forbidden from .meta import Path, HasPaths, expand_template, GrondError, Forbidden
...@@ -130,7 +131,8 @@ class Config(HasPaths): ...@@ -130,7 +131,8 @@ class Config(HasPaths):
def get_problem(self, event): def get_problem(self, event):
targets = self.get_targets(event) targets = self.get_targets(event)
problem = self.problem_config.get_problem(event, targets) problem = self.problem_config.get_problem(
event, self.target_groups, targets)
self.setup_modelling_environment(problem) self.setup_modelling_environment(problem)
return problem return problem
...@@ -254,7 +256,7 @@ def forward(rundir_or_config_path, event_names): ...@@ -254,7 +256,7 @@ def forward(rundir_or_config_path, event_names):
events.append(event) events.append(event)
for result in results: for result in results:
if not isinstance(result, gf.SeismosizerError): if isinstance(result, WaveformMisfitResult):
result.filtered_obs.set_codes(location='ob') result.filtered_obs.set_codes(location='ob')
result.filtered_syn.set_codes(location='sy') result.filtered_syn.set_codes(location='sy')
all_trs.append(result.filtered_obs) all_trs.append(result.filtered_obs)
...@@ -452,6 +454,7 @@ def check( ...@@ -452,6 +454,7 @@ def check(
backazimuth=target. backazimuth=target.
get_backazimuth_for_waveform(), get_backazimuth_for_waveform(),
debug=True) debug=True)
except NotFound as e: except NotFound as e:
logger.warn(str(e)) logger.warn(str(e))
continue continue
...@@ -498,7 +501,7 @@ def check( ...@@ -498,7 +501,7 @@ def check(
yabsmaxs = [] yabsmaxs = []
for results in results_list: for results in results_list:
result = results[itarget] result = results[itarget]
if not isinstance(result, gf.SeismosizerError): if isinstance(result, WaveformMisfitResult):
yabsmaxs.append( yabsmaxs.append(
num.max(num.abs( num.max(num.abs(
result.filtered_obs.get_ydata()))) result.filtered_obs.get_ydata())))
...@@ -512,7 +515,7 @@ def check( ...@@ -512,7 +515,7 @@ def check(
ii = 0 ii = 0
for results in results_list: for results in results_list:
result = results[itarget] result = results[itarget]
if not isinstance(result, gf.SeismosizerError): if isinstance(result, WaveformMisfitResult):
if fig is None: if fig is None:
fig = plt.figure() fig = plt.figure()
axes = fig.add_subplot(1, 1, 1) axes = fig.add_subplot(1, 1, 1)
...@@ -521,7 +524,8 @@ def check( ...@@ -521,7 +524,8 @@ def check(
xdata = result.filtered_obs.get_xdata() xdata = result.filtered_obs.get_xdata()
ydata = result.filtered_obs.get_ydata() / yabsmax ydata = result.filtered_obs.get_ydata() / yabsmax
axes.plot(xdata, ydata*0.5 + 3.5, color='black') axes.plot(
xdata, ydata*0.5 + 3.5, color='black')
color = plot.mpl_graph_color(ii) color = plot.mpl_graph_color(ii)
......
...@@ -442,8 +442,8 @@ class HighScoreOptimizer(Optimizer): ...@@ -442,8 +442,8 @@ class HighScoreOptimizer(Optimizer):
isbad_mask != isbad_mask_new): isbad_mask != isbad_mask_new):
errmess = [ errmess = [
'problem %s: inconsistency in data availability' % 'problem %s: inconsistency in data availability at iteration %i' %
problem.name] (problem.name, iiter)]
for target, isbad_new, isbad in zip( for target, isbad_new, isbad in zip(
problem.targets, isbad_mask_new, isbad_mask): problem.targets, isbad_mask_new, isbad_mask):
...@@ -452,7 +452,7 @@ class HighScoreOptimizer(Optimizer): ...@@ -452,7 +452,7 @@ class HighScoreOptimizer(Optimizer):
errmess.append(' %s, %s -> %s' % ( errmess.append(' %s, %s -> %s' % (
target.string_id(), isbad, isbad_new)) target.string_id(), isbad, isbad_new))
raise BadProblem(errmess) raise BadProblem('\n'.join(errmess))
isbad_mask = isbad_mask_new isbad_mask = isbad_mask_new
......
...@@ -2086,6 +2086,10 @@ def plot_result(dirname, plotnames_want, ...@@ -2086,6 +2086,10 @@ def plot_result(dirname, plotnames_want,
fns.extend( fns.extend(
save_figs(figs, plot_dirname, plotname, formats, dpi)) save_figs(figs, plot_dirname, plotname, formats, dpi))
for fig in figs:
plt.close(fig)
if 7 != len({ if 7 != len({
'fits', 'fits',
'fits_statics', 'fits_statics',
...@@ -2114,6 +2118,9 @@ def plot_result(dirname, plotnames_want, ...@@ -2114,6 +2118,9 @@ def plot_result(dirname, plotnames_want,
fns.extend( fns.extend(
save_figs(figs, plot_dirname, plotname, formats, dpi)) save_figs(figs, plot_dirname, plotname, formats, dpi))
for fig in figs:
plt.close(fig)
for plotname in ['jointpar', 'hudson', 'solution', 'location']: for plotname in ['jointpar', 'hudson', 'solution', 'location']:
if plotname in plotnames_want: if plotname in plotnames_want:
figs = plot_dispatch[plotname](history, optimizer, plt) figs = plot_dispatch[plotname](history, optimizer, plt)
...@@ -2121,6 +2128,9 @@ def plot_result(dirname, plotnames_want, ...@@ -2121,6 +2128,9 @@ def plot_result(dirname, plotnames_want,
fns.extend( fns.extend(
save_figs(figs, plot_dirname, plotname, formats, dpi)) save_figs(figs, plot_dirname, plotname, formats, dpi))
for fig in figs:
plt.close(fig)
if not save: if not save:
plt.show() plt.show()
......
...@@ -11,7 +11,8 @@ from pyrocko import gf, util, guts ...@@ -11,7 +11,8 @@ from pyrocko import gf, util, guts
from pyrocko.guts import Object, String, Bool, List, Dict, Int from pyrocko.guts import Object, String, Bool, List, Dict, Int
from ..meta import ADict, Parameter, GrondError, xjoin from ..meta import ADict, Parameter, GrondError, xjoin
from ..targets import WaveformMisfitTarget, SatelliteMisfitTarget from ..targets import MisfitTarget, TargetGroup, WaveformMisfitTarget, \
SatelliteMisfitTarget
guts_prefix = 'grond' guts_prefix = 'grond'
...@@ -37,8 +38,8 @@ class Problem(Object): ...@@ -37,8 +38,8 @@ class Problem(Object):
apply_balancing_weights = Bool.T(default=True) apply_balancing_weights = Bool.T(default=True)
norm_exponent = Int.T(default=2) norm_exponent = Int.T(default=2)
base_source = gf.Source.T(optional=True) base_source = gf.Source.T(optional=True)
targets = List.T(MisfitTarget.T())
targets = List.T() target_groups = List.T(TargetGroup.T())
def __init__(self, **kwargs): def __init__(self, **kwargs):
Object.__init__(self, **kwargs) Object.__init__(self, **kwargs)
...@@ -212,10 +213,10 @@ class Problem(Object): ...@@ -212,10 +213,10 @@ class Problem(Object):
def get_target_weights(self): def get_target_weights(self):
if self._target_weights is None: if self._target_weights is None:
self._target_weights = num.array( self._target_weights = num.concatenate(
[target.get_combined_weight( [target.get_combined_weight(
apply_balancing_weights=self.apply_balancing_weights) apply_balancing_weights=self.apply_balancing_weights)
for target in self.targets], dtype=num.float) for target in self.targets])
return self._target_weights return self._target_weights
...@@ -352,26 +353,47 @@ class Problem(Object): ...@@ -352,26 +353,47 @@ class Problem(Object):
return self._family_mask return self._family_mask
def evaluate(self, x, dump_data=False): def evaluate(self, x, mask=None, result_mode='sparse'):
raise NotImplementedError()
def forward(self, x):
source = self.get_source(x) source = self.get_source(x)
engine = self.get_engine() engine = self.get_engine()
plain_targets = [target.get_plain_target() for target in self.targets]
resp = engine.process(source, plain_targets) for target in self.targets:
target.set_result_mode(result_mode)
modelling_targets = []
t2m_map = {}
for itarget, target in enumerate(self.targets):
t2m_map[target] = target.prepare_modelling()
if mask is None or mask[itarget]:
modelling_targets.extend(t2m_map[target])
resp = engine.process(source, modelling_targets)
modelling_results = list(resp.results_list[0])
imt = 0
imisfit = 0
misfits = num.zeros((self.nmisfits, 2))
misfits.fill(None)
results = [] results = []
for target, result in zip(self.targets, resp.results_list[0]): for itarget, target in enumerate(self.targets):
if isinstance(result, gf.SeismosizerError): nmt_this = len(t2m_map[target])
logger.debug( if mask is None or mask[itarget]:
'%s.%s.%s.%s: %s' % (target.codes + (str(result),))) misfits[imisfit:imisfit+target.nmisfits, :], result = \
target.finalize_modelling(
modelling_results[imt:imt+nmt_this])
results.append(None) imt += nmt_this
else: else:
results.append(result) result = gf.SeismosizerError(
'target was excluded from modelling')
return results results.append(result)
imisfit += target.nmisfits
if result_mode == 'full':
return misfits, results
else:
return misfits
class InvalidRundir(Exception): class InvalidRundir(Exception):
......
...@@ -21,7 +21,7 @@ class CMTProblemConfig(ProblemConfig): ...@@ -21,7 +21,7 @@ class CMTProblemConfig(ProblemConfig):
distance_min = Float.T(default=0.0) distance_min = Float.T(default=0.0)
mt_type = StringChoice.T(choices=['full', 'deviatoric']) mt_type = StringChoice.T(choices=['full', 'deviatoric'])
def get_problem(self, event, targets): def get_problem(self, event, target_groups, targets):
if event.depth is None: if event.depth is None:
event.depth = 0. event.depth = 0.
...@@ -36,6 +36,7 @@ class CMTProblemConfig(ProblemConfig): ...@@ -36,6 +36,7 @@ class CMTProblemConfig(ProblemConfig):
name=expand_template(self.name_template, subs), name=expand_template(self.name_template, subs),
apply_balancing_weights=self.apply_balancing_weights, apply_balancing_weights=self.apply_balancing_weights,
base_source=base_source, base_source=base_source,
target_groups=target_groups,
targets=targets, targets=targets,
ranges=self.ranges, ranges=self.ranges,
distance_min=self.distance_min, distance_min=self.distance_min,
...@@ -206,70 +207,6 @@ class CMTProblem(Problem): ...@@ -206,70 +207,6 @@ class CMTProblem(Problem):
return out return out
def evaluate(self, x, result_mode='sparse', mask=None):
source = self.get_source(x)
engine = self.get_engine()
for target in self.targets:
target.set_result_mode(result_mode)
if mask is not None:
assert len(mask) == len(self.targets)
targets_ok = [
target for (target, ok) in zip(self.targets, mask) if ok]
else:
targets_ok = self.targets
resp = engine.process(source, targets_ok)
if mask is not None:
ires_ok = 0
results = []
for target, ok in zip(self.targets, mask):
if ok:
results.append(resp.results_list[0][ires_ok])
ires_ok += 1
else:
results.append(
gf.SeismosizerError(
'skipped because of previous failure'))
else:
results = list(resp.results_list[0])
data = []
for target, result in zip(self.targets, results):
if isinstance(result, gf.SeismosizerError):
logger.debug(
'%s.%s.%s.%s: %s' % (target.codes + (str(result),)))
data.append((None, None))
else:
data.append((result.misfit_value, result.misfit_norm))
misfits = num.array(data, dtype=num.float)
if result_mode == 'full':
return misfits, results
else:
return misfits
def forward(self, x):
source = self.get_source(x)
engine = self.get_engine()
plain_targets = [target.get_plain_target() for target in self.targets]
resp = engine.process(source, plain_targets)
results = []
for target, result in zip(self.targets, resp.results_list[0]):
if isinstance(result, gf.SeismosizerError):
logger.debug(
'%s.%s.%s.%s: %s' % (target.codes + (str(result),)))
results.append(None)
else:
results.append(result)
return results
__all__ = ''' __all__ = '''
CMTProblem CMTProblem
......
...@@ -126,52 +126,6 @@ class DoubleDCProblem(Problem): ...@@ -126,52 +126,6 @@ class DoubleDCProblem(Problem):
return num.array(x, dtype=num.float) return num.array(x, dtype=num.float)
def evaluate(self, x, result_mode='sparse'):
source = self.get_source(x)
engine = self.get_engine()
for target in self.targets:
target.set_result_mode(result_mode)
resp = engine.process(source, self.targets)
data = []
results = []
for target, result in zip(self.targets, resp.results_list[0]):
if isinstance(result, gf.SeismosizerError):
logger.debug(
'%s.%s.%s.%s: %s' % (target.codes + (str(result),)))
data.append((None, None))
results.append(result)
else:
data.append((result.misfit_value, result.misfit_norm))
results.append(result)
ms, ns = num.array(data, dtype=num.float).T
if result_mode == 'full':
return ms, ns, results
else:
return ms, ns
def forward(self, x):
source = self.get_source(x)
engine = self.get_engine()
plain_targets = [target.get_plain_target() for target in self.targets]
resp = engine.process(source, plain_targets)
results = []
for target, result in zip(self.targets, resp.results_list[0]):
if isinstance(result, gf.SeismosizerError):
logger.debug(
'%s.%s.%s.%s: %s' % (target.codes + (str(result),)))
results.append(None)
else:
results.append(result)
return results
__all__ = ''' __all__ = '''
DoubleDCProblem DoubleDCProblem
......
...@@ -99,70 +99,6 @@ class RectangularProblem(Problem): ...@@ -99,70 +99,6 @@ class RectangularProblem(Problem):
# raise Forbidden() # raise Forbidden()
return x return x
def evaluate(self, x, result_mode='sparse', mask=None, nprocs=0):
source = self.get_source(x)
engine = self.get_engine()
for target in self.targets:
target.set_result_mode(result_mode)
if mask is not None:
assert len(mask) == len(self.targets)
targets_ok = [
target for (target, ok) in zip(self.targets, mask) if ok]
else:
targets_ok = self.targets
self.set_target_parameter_values(x)
resp = engine.process(source, targets_ok, nprocs=nprocs)
if mask is not None:
ires_ok = 0
results = []
for target, ok in zip(self.targets, mask):
if ok:
results.append(resp.results_list[0][ires_ok])
ires_ok += 1
else:
results.append(
gf.SeismosizerError(
'skipped because of previous failure'))
else:
results = list(resp.results_list[0])
data = []
for target, result in zip(self.targets, results):
if isinstance(result, gf.SeismosizerError):
logger.debug(
'%s.%s.%s.%s: %s' % (target.codes + (str(result),)))
data.append((None, None))
else:
data.append((result.misfit_value, result.misfit_norm))
ms, ns = num.array(data, dtype=num.float).T
if result_mode == 'full':
return ms, ns, results
else:
return ms, ns
def forward(self, x, nprocs=0):
source = self.get_source(x)
engine = self.get_engine()
plain_targets = [target.get_plain_target() for target in self.targets]
resp = engine.process(source, plain_targets, nprocs=nprocs)
results = []
for target, result in zip(self.targets, resp.results_list[0]):
if isinstance(result, gf.SeismosizerError):
logger.debug(
'%s.%s.%s.%s: %s' % (target.codes + (str(result),)))
results.append(None)
else:
results.append(result)
return results
__all__ = ''' __all__ = '''
RectangularProblem RectangularProblem
......
import copy import copy
import numpy as num
from pyrocko import gf from pyrocko import gf
from pyrocko.guts import Object, Float from pyrocko.guts import Object, Float
...@@ -7,17 +9,11 @@ from pyrocko.guts import Object, Float ...@@ -7,17 +9,11 @@ from pyrocko.guts import Object, Float
guts_prefix = 'grond' guts_prefix = 'grond'
class MisfitConfig(Object):
pass
class TargetGroup(Object): class TargetGroup(Object):
normalisation_family = gf.StringID.T(optional=True) normalisation_family = gf.StringID.T(optional=True)
path = gf.StringID.T(optional=True) path = gf.StringID.T(optional=True)
weight = Float.T(default=1.0) weight = Float.T(default=1.0)
misfit_config = MisfitConfig.T(optional=True)
interpolation = gf.InterpolationMethod.T() interpolation = gf.InterpolationMethod.T()
store_id = gf.StringID.T(optional=True) store_id = gf.StringID.T(optional=True)
...@@ -32,9 +28,8 @@ class TargetAnalysisResult(Object): ...@@ -32,9 +28,8 @@ class TargetAnalysisResult(Object):
balancing_weight = Float.T() balancing_weight = Float.T()
class MisfitResult(gf.Result): class MisfitResult(Object):
misfit_value = Float.T() pass
misfit_norm = Float.T()
class MisfitTarget(Object): class MisfitTarget(Object):
...@@ -44,19 +39,15 @@ class MisfitTarget(Object): ...@@ -44,19 +39,15 @@ class MisfitTarget(Object):
help='Relative weight of this target') help='Relative weight of this target')
analysis_result = TargetAnalysisResult.T( analysis_result = TargetAnalysisResult.T(
optional=True) optional=True)
misfit_config = MisfitConfig.T(
optional=True,
help='Configuration object how the misfit is calculated.')
normalisation_family = gf.StringID.T( normalisation_family = gf.StringID.T(
optional=True, optional=True,
help='Normalisation family of this misfitTarget')