Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Sebastian Heimann
grond
Commits
5d1cbdbd
Commit
5d1cbdbd
authored
Oct 14, 2016
by
Sebastian Heimann
Browse files
restructuring for better handling of event datasets and synthetic tests
parent
cb04b480
Changes
5
Hide whitespace changes
Inline
Side-by-side
apps/grond
View file @
5d1cbdbd
...
...
@@ -34,9 +34,11 @@ subcommand_descriptions = {
subcommand_usages
=
{
'init'
:
'init [options]'
,
'check'
:
'check <configfile> [options]'
,
'go'
:
'go <configfile> [options]'
,
'forward'
:
'forward <rundir> [options]'
,
'check'
:
'check <configfile> <eventnames> ... [options]'
,
'go'
:
'go <configfile> <eventnames> ... [options]'
,
'forward'
:
(
'forward <rundir> [options]'
,
'forward <configfile> <eventnames> ... [options]'
),
'harvest'
:
'harvest <rundir> [options]'
,
'plot'
:
'plot <plotnames> <rundir> [options]'
,
'export'
:
'export (best|mean|ensemble|stats) <rundirs> ... [options]'
,
...
...
@@ -173,19 +175,15 @@ def command_init(args):
def
command_check
(
args
):
def
setup
(
parser
):
parser
.
add_option
(
'--event'
,
dest
=
'event_name'
,
metavar
=
'EVENTNAME'
,
help
=
'process only event EVENTNAME'
)
pass
parser
,
options
,
args
=
cl_parse
(
'check'
,
args
,
setup
)
if
len
(
args
)
!=
1
:
help_and_die
(
parser
,
'no config file given'
)
event_names
=
None
if
options
.
event_name
:
event_names
=
[
options
.
event_name
]
if
len
(
args
)
<
2
:
help_and_die
(
parser
,
'missing arguments'
)
config_path
=
args
[
0
]
event_names
=
args
[
1
:]
config
=
grond
.
read_config
(
config_path
)
grond
.
check
(
...
...
@@ -195,9 +193,6 @@ def command_check(args):
def
command_go
(
args
):
def
setup
(
parser
):
parser
.
add_option
(
'--event'
,
dest
=
'event_name'
,
metavar
=
'EVENTNAME'
,
help
=
'process only event EVENTNAME'
)
parser
.
add_option
(
'--force'
,
dest
=
'force'
,
action
=
'store_true'
,
help
=
'overwrite existing run directory'
)
...
...
@@ -209,20 +204,18 @@ def command_go(args):
help
=
'set number of events to process in parallel'
)
parser
,
options
,
args
=
cl_parse
(
'go'
,
args
,
setup
)
if
len
(
args
)
!=
1
:
help_and_die
(
parser
,
'
no config file given
'
)
if
len
(
args
)
<
2
:
help_and_die
(
parser
,
'
missing arguments
'
)
config_path
=
args
[
0
]
event_names
=
args
[
1
:]
config
=
grond
.
read_config
(
config_path
)
if
options
.
status
==
'quiet'
:
status
=
()
else
:
status
=
tuple
(
options
.
status
.
split
(
','
))
event_names
=
None
if
options
.
event_name
:
event_names
=
[
options
.
event_name
]
grond
.
go
(
config
,
event_names
=
event_names
,
...
...
@@ -233,19 +226,15 @@ def command_go(args):
def
command_forward
(
args
):
def
setup
(
parser
):
parser
.
add_option
(
'--event'
,
dest
=
'event_name'
,
metavar
=
'EVENTNAME'
,
help
=
'process only event EVENTNAME'
)
pass
parser
,
options
,
args
=
cl_parse
(
'forward'
,
args
,
setup
)
if
len
(
args
)
!=
1
:
help_and_die
(
parser
,
'
incorrect number of
arguments'
)
if
len
(
args
)
<
1
:
help_and_die
(
parser
,
'
missing
arguments'
)
event_names
=
None
if
options
.
event_name
:
event_names
=
[
options
.
event_name
]
event_names
=
args
[
1
:]
run_path
,
=
args
run_path
=
args
[
0
]
grond
.
forward
(
run_path
,
event_names
=
event_names
)
...
...
src/cmt.py
View file @
5d1cbdbd
...
...
@@ -37,7 +37,6 @@ class CMTProblem(core.Problem):
core
.
Parameter
(
'rel_moment_iso'
,
label
=
'$M_{0}^{ISO}/M_{0}$'
),
core
.
Parameter
(
'rel_moment_clvd'
,
label
=
'$M_{0}^{CLVD}/M_{0}$'
)]
base_source
=
gf
.
Source
.
T
()
targets
=
List
.
T
(
core
.
MisfitTarget
.
T
())
ranges
=
Dict
.
T
(
String
.
T
(),
gf
.
Range
.
T
())
...
...
@@ -303,7 +302,7 @@ class CMTProblemConfig(core.ProblemConfig):
event_time
=
util
.
time_to_str
(
event
.
time
))
problem
=
CMTProblem
(
name
=
self
.
name_template
%
subs
,
name
=
core
.
substitute_template
(
self
.
name_template
,
subs
)
,
apply_balancing_weights
=
self
.
apply_balancing_weights
,
base_source
=
base_source
,
targets
=
targets
,
...
...
src/core.py
View file @
5d1cbdbd
...
...
@@ -6,6 +6,7 @@ import time
import
copy
import
shutil
import
os.path
as
op
from
string
import
Template
import
numpy
as
num
...
...
@@ -122,6 +123,7 @@ class Problem(Object):
parameters
=
List
.
T
(
Parameter
.
T
())
dependants
=
List
.
T
(
Parameter
.
T
())
apply_balancing_weights
=
Bool
.
T
(
default
=
True
)
base_source
=
gf
.
Source
.
T
()
def
__init__
(
self
,
**
kwargs
):
Object
.
__init__
(
self
,
**
kwargs
)
...
...
@@ -666,45 +668,27 @@ class SyntheticWaveformNotAvailable(Exception):
class
SyntheticTest
(
Object
):
random_seed
=
Int
.
T
(
default
=
0
)
inject_solution
=
Bool
.
T
(
default
=
False
)
ignore
_data_availability
=
Bool
.
T
(
default
=
False
)
respect
_data_availability
=
Bool
.
T
(
default
=
False
)
add_real_noise
=
Bool
.
T
(
default
=
False
)
toffset_real_noise
=
Float
.
T
(
default
=-
3600.
)
x
=
Dict
.
T
(
String
.
T
(),
Float
.
T
())
def
__init__
(
self
,
**
kwargs
):
Object
.
__init__
(
self
,
**
kwargs
)
self
.
_problem
=
None
self
.
_synthetics
=
None
self
.
_rstate
=
num
.
random
.
RandomState
(
self
.
random_seed
)
def
set_config
(
self
,
config
):
self
.
_config
=
config
def
set_problem
(
self
,
problem
):
self
.
_problem
=
problem
self
.
_synthetics
=
None
def
get_problem
(
self
):
ds
=
self
.
_config
.
get_dataset
()
events
=
ds
.
get_events
()
event
=
events
[
0
]
return
self
.
_config
.
get_problem
(
event
)
def
get_x_random
(
self
):
problem
=
self
.
get_problem
()
xbounds
=
num
.
array
(
problem
.
bounds
(),
dtype
=
num
.
float
)
npar
=
xbounds
.
shape
[
0
]
x
=
num
.
zeros
(
npar
,
dtype
=
num
.
float
)
while
True
:
for
i
in
xrange
(
npar
):
x
[
i
]
=
self
.
_rstate
.
uniform
(
xbounds
[
i
,
0
],
xbounds
[
i
,
1
])
try
:
x
=
problem
.
preconstrain
(
x
)
break
except
Forbidden
:
pass
if
self
.
_problem
is
None
:
raise
SyntheticWaveformNotAvailable
(
'SyntheticTest.set_problem() has not been called yet'
)
return
x
return
self
.
_problem
def
get_x
(
self
):
problem
=
self
.
get_problem
()
...
...
@@ -713,22 +697,16 @@ class SyntheticTest(Object):
problem
.
parameter_array
(
self
.
x
))
else
:
print
problem
.
base_source
x
=
problem
.
preconstrain
(
problem
.
pack
(
problem
.
base_source
))
print
x
print
problem
.
unpack
(
x
)
return
x
def
get_synthetics
(
self
):
problem
=
self
.
get_problem
()
if
self
.
_synthetics
is
None
:
problem
=
self
.
get_problem
()
x
=
self
.
get_x
()
results
=
problem
.
forward
(
x
)
self
.
_synthetics
=
results
...
...
@@ -775,12 +753,12 @@ class DatasetConfig(HasPaths):
def
__init__
(
self
,
*
args
,
**
kwargs
):
HasPaths
.
__init__
(
self
,
*
args
,
**
kwargs
)
self
.
_ds
=
None
self
.
_ds
=
{}
def
get_dataset
(
self
):
if
self
.
_ds
is
None
:
def
get_dataset
(
self
,
event_name
):
if
event_name
not
in
self
.
_ds
:
fp
=
self
.
expand_path
ds
=
dataset
.
Dataset
()
ds
=
dataset
.
Dataset
(
event_name
)
ds
.
add_stations
(
pyrocko_stations_filename
=
fp
(
self
.
stations_path
),
stationxml_filenames
=
fp
(
self
.
stations_stationxml_paths
))
...
...
@@ -813,10 +791,10 @@ class DatasetConfig(HasPaths):
if
self
.
whitelist
:
ds
.
add_whitelist
(
self
.
whitelist
)
ds
.
set_synthetic_test
(
self
.
synthetic_test
)
self
.
_ds
=
ds
ds
.
set_synthetic_test
(
copy
.
deepcopy
(
self
.
synthetic_test
)
)
self
.
_ds
[
event_name
]
=
ds
return
self
.
_ds
return
self
.
_ds
[
event_name
]
def
weed
(
origin
,
targets
,
limit
,
neighborhood
=
3
):
...
...
@@ -961,15 +939,11 @@ class Config(HasPaths):
def
__init__
(
self
,
*
args
,
**
kwargs
):
HasPaths
.
__init__
(
self
,
*
args
,
**
kwargs
)
def
get_dataset
(
self
):
ds
=
self
.
dataset_config
.
get_dataset
()
if
ds
.
synthetic_test
:
ds
.
synthetic_test
.
set_config
(
self
)
return
ds
def
get_dataset
(
self
,
event_name
):
return
self
.
dataset_config
.
get_dataset
(
event_name
)
def
get_targets
(
self
,
event
):
ds
=
self
.
get_dataset
()
ds
=
self
.
get_dataset
(
event
.
name
)
targets
=
[]
for
igroup
,
target_config
in
enumerate
(
self
.
target_configs
):
...
...
@@ -978,10 +952,18 @@ class Config(HasPaths):
return
targets
def
setup_modelling_environment
(
self
,
problem
):
problem
.
set_engine
(
self
.
engine_config
.
get_engine
())
ds
=
self
.
get_dataset
(
problem
.
base_source
.
name
)
synt
=
ds
.
synthetic_test
if
synt
:
synt
.
set_problem
(
problem
)
problem
.
base_source
=
problem
.
unpack
(
synt
.
get_x
())
def
get_problem
(
self
,
event
):
targets
=
self
.
get_targets
(
event
)
problem
=
self
.
problem_config
.
get_problem
(
event
,
targets
)
problem
.
set_engine
(
self
.
engine_config
.
get_engine
()
)
self
.
setup_modelling_environment
(
problem
)
return
problem
...
...
@@ -1449,7 +1431,7 @@ def forward(rundir_or_config_path, event_names=None):
ibest
=
num
.
argmin
(
gms
)
xbest
=
xs
[
ibest
,
:]
ds
=
config
.
get_dataset
()
ds
=
config
.
get_dataset
(
problem
.
base_source
.
name
)
problem
.
set_engine
(
config
.
engine_config
.
get_engine
())
for
target
in
problem
.
targets
:
...
...
@@ -1459,11 +1441,11 @@ def forward(rundir_or_config_path, event_names=None):
else
:
config
=
read_config
(
rundir_or_config_path
)
ds
=
config
.
get_dataset
()
events
=
ds
.
get_events
(
event_names
=
event_names
)
payload
=
[]
for
event
in
events
:
for
event_name
in
event_names
:
ds
=
config
.
get_dataset
(
event_name
)
event
=
ds
.
get_event
()
problem
=
config
.
get_problem
(
event
)
xref
=
problem
.
preconstrain
(
problem
.
pack
(
problem
.
base_source
))
...
...
@@ -1556,17 +1538,12 @@ g_state = {}
def
check
(
config
,
event_names
=
None
):
ds
=
config
.
get_dataset
()
events
=
ds
.
get_events
(
event_names
=
event_names
)
nevents
=
len
(
events
)
from
matplotlib
import
pyplot
as
plt
from
grond.plot
import
colors
if
nevents
==
0
:
raise
GrondError
(
'no events found'
)
for
ievent
,
event
in
enumerate
(
events
):
for
ievent
,
event_name
in
enumerate
(
event_names
):
ds
=
config
.
get_dataset
(
event_name
)
event
=
ds
.
get_event
()
try
:
problem
=
config
.
get_problem
(
event
)
check_problem
(
problem
)
...
...
@@ -1651,18 +1628,12 @@ def go(config, event_names=None, force=False, nparallel=1, status=('state',)):
status
=
tuple
(
status
)
ds
=
config
.
get_dataset
()
events
=
ds
.
get_events
(
event_names
=
event_names
)
nevents
=
len
(
events
)
if
nevents
==
0
:
raise
GrondError
(
'no events found'
)
g_data
=
(
config
,
force
,
status
,
nparallel
,
event_names
)
g_state
[
id
(
g_data
)]
=
g_data
nevents
=
len
(
event_names
)
for
x
in
parimap
.
parimap
(
process_event
,
xrange
(
nevents
),
...
...
@@ -1672,6 +1643,17 @@ def go(config, event_names=None, force=False, nparallel=1, status=('state',)):
pass
def
substitute_template
(
template
,
d
):
try
:
return
Template
(
template
).
substitute
(
d
)
except
KeyError
as
e
:
raise
GrondError
(
'invalid placeholder "%s" in template: "%s"'
%
(
str
(
e
),
template
))
except
ValueError
:
raise
GrondError
(
'malformed placeholder in template: "%s"'
%
template
)
def
process_event
(
ievent
,
g_data_id
):
config
,
force
,
status
,
nparallel
,
event_names
=
g_state
[
g_data_id
]
...
...
@@ -1679,28 +1661,27 @@ def process_event(ievent, g_data_id):
if
nparallel
>
1
:
status
=
()
ds
=
config
.
get_dataset
()
events
=
ds
.
get_events
(
event_names
=
event_names
)
nevents
=
len
(
events
)
event_name
=
event_names
[
ievent
]
event
=
events
[
ievent
]
ds
=
config
.
get_dataset
(
event_name
)
ds
.
empty_cache
(
)
nevents
=
len
(
event_names
)
tstart
=
time
.
time
()
event
=
ds
.
get_event
()
problem
=
config
.
get_problem
(
event
)
# FIXME
synt
=
ds
.
synthetic_test
if
synt
and
synt
.
inject_solution
:
if
synt
:
problem
.
base_source
=
problem
.
unpack
(
synt
.
get_x
())
check_problem
(
problem
)
rundir
=
config
.
rundir_template
%
dict
(
problem_name
=
problem
.
name
)
rundir
=
substitute_template
(
config
.
rundir_template
,
dict
(
problem_name
=
problem
.
name
))
if
op
.
exists
(
rundir
):
if
force
:
...
...
src/dataset.py
View file @
5d1cbdbd
...
...
@@ -53,7 +53,7 @@ def dump_station_corrections(station_corrections, filename):
class
Dataset
(
object
):
def
__init__
(
self
):
def
__init__
(
self
,
event_name
=
None
):
self
.
events
=
[]
self
.
pile
=
pile
.
Pile
()
self
.
stations
=
{}
...
...
@@ -72,6 +72,7 @@ class Dataset(object):
self
.
synthetic_test
=
None
self
.
_picks
=
None
self
.
_cache
=
{}
self
.
_event_name
=
event_name
def
empty_cache
(
self
):
self
.
_cache
=
{}
...
...
@@ -493,15 +494,16 @@ class Dataset(object):
syn_test
=
self
.
synthetic_test
toffset_noise_extract
=
0.0
if
syn_test
:
if
syn_test
.
ignore
_data_availability
:
if
not
syn_test
.
respect
_data_availability
:
if
syn_test
.
add_real_noise
:
raise
DatasetError
(
'
ignore
_data_availability=
Tru
e and '
'
respect
_data_availability=
Fals
e and '
'add_real_noise=True cannot be combined.'
)
tr
=
syn_test
.
get_waveform
(
nslc
,
tmin
,
tmax
,
tfade
=
tfade
,
freqlimits
=
freqlimits
)
tfade
=
tfade
,
freqlimits
=
freqlimits
)
if
cache
is
not
None
:
cache
[
tr
.
nslc_id
,
tmin
,
tmax
]
=
tr
...
...
@@ -595,7 +597,7 @@ class Dataset(object):
return
evs
def
get_event
(
self
,
t
,
magmin
=
None
):
def
get_event
_by_time
(
self
,
t
,
magmin
=
None
):
evs
=
self
.
get_events
(
magmin
=
magmin
)
ev_x
=
None
for
ev
in
evs
:
...
...
@@ -609,6 +611,16 @@ class Dataset(object):
return
ev_x
def
get_event
(
self
):
if
self
.
_event_name
is
None
:
raise
NotFound
(
'no main event selected in dataset'
)
for
ev
in
self
.
events
:
if
ev
.
name
==
self
.
_event_name
:
return
ev
raise
NotFound
(
'no such event: %s'
%
self
.
_event_name
)
def
get_picks
(
self
):
if
self
.
_picks
is
None
:
hash_to_name
=
{}
...
...
src/plot.py
View file @
5d1cbdbd
...
...
@@ -1458,8 +1458,9 @@ def plot_result(dirname, plotnames_want,
if
plotname
in
plotnames_want
:
config
=
guts
.
load
(
filename
=
op
.
join
(
dirname
,
'config.yaml'
))
config
.
set_basepath
(
dirname
)
problem
.
set_engine
(
config
.
engine_config
.
get_engine
())
ds
=
config
.
get_dataset
()
config
.
setup_modelling_environment
(
problem
)
event_name
=
problem
.
base_source
.
name
ds
=
config
.
get_dataset
(
event_name
)
figs
=
plot_dispatch
[
plotname
](
ds
,
model
,
plt
)
if
save
:
fns
.
extend
(
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment