Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Sebastian Heimann
grond
Commits
d36cce1c
Commit
d36cce1c
authored
Nov 28, 2017
by
Marius Isken
Browse files
adapting new state
parent
fe362a65
Changes
8
Hide whitespace changes
Inline
Side-by-side
src/analysers/base.py
View file @
d36cce1c
import
copy
import
numpy
as
num
from
pyrocko
import
util
from
pyrocko.guts
import
Object
,
Int
from
..targets
import
TargetAnalysisResult
from
..meta
import
Forbidden
...
...
@@ -12,8 +14,9 @@ class Analyser(object):
def
__init__
(
self
,
niter
):
self
.
niter
=
niter
self
.
pbar
=
util
.
progressbar
(
'analysing problem'
,
niter
)
def
analyse
(
self
,
problem
,
notifier
):
def
analyse
(
self
,
problem
):
if
self
.
niter
==
0
:
return
...
...
@@ -36,10 +39,11 @@ class Analyser(object):
mss
=
num
.
zeros
((
self
.
niter
,
wproblem
.
ntargets
))
rstate
=
num
.
random
.
RandomState
(
123
)
notifier
.
emit
(
'progress_start'
,
'analysing problem'
,
self
.
niter
)
isbad_mask
=
None
self
.
pbar
.
start
()
for
iiter
in
range
(
self
.
niter
):
self
.
pbar
.
update
(
iiter
)
while
True
:
x
=
[]
for
ipar
in
range
(
npar
):
...
...
@@ -61,9 +65,7 @@ class Analyser(object):
mss
[
iiter
,
:]
=
ms
isbad_mask
=
num
.
isnan
(
ms
)
notifier
.
emit
(
'progress_update'
,
'analysing problem'
,
iiter
)
notifier
.
emit
(
'progress_finish'
,
'analysing problem'
)
self
.
pbar
.
finish
()
mean_ms
=
num
.
mean
(
mss
,
axis
=
0
)
weights
=
1.
/
mean_ms
...
...
src/core.py
View file @
d36cce1c
...
...
@@ -20,14 +20,27 @@ from .problems.base import ProblemConfig, Problem, \
from
.optimizers.base
import
OptimizerConfig
,
BadProblem
from
.targets.base
import
TargetGroup
from
.analysers.base
import
AnalyserConfig
from
.listeners
import
TerminalListener
from
.meta
import
Path
,
HasPaths
,
expand_template
,
GrondError
,
Notifier
,
\
Forbidden
from
.meta
import
Path
,
HasPaths
,
expand_template
,
GrondError
,
Forbidden
logger
=
logging
.
getLogger
(
'grond.core'
)
guts_prefix
=
'grond'
class
RingBuffer
(
num
.
ndarray
):
def
__new__
(
cls
,
*
args
,
**
kwargs
):
cls
=
num
.
ndarray
.
__new__
(
cls
,
*
args
,
**
kwargs
)
cls
.
fill
(
0.
)
return
cls
def
__init__
(
self
,
*
args
,
**
kwargs
):
self
.
pos
=
0
def
put
(
self
,
value
):
self
[
self
.
pos
]
=
value
self
.
pos
+=
1
self
.
pos
%=
self
.
size
def
mahalanobis_distance
(
xs
,
mx
,
cov
):
imask
=
num
.
diag
(
cov
)
!=
0.
icov
=
num
.
linalg
.
inv
(
cov
[
imask
,
:][:,
imask
])
...
...
@@ -633,11 +646,8 @@ def process_event(ievent, g_data_id):
logger
.
info
(
'start %i / %i'
%
(
ievent
+
1
,
nevents
))
notifier
=
Notifier
()
notifier
.
add_listener
(
TerminalListener
())
analyser
=
config
.
analyser_config
.
get_analyser
()
analyser
.
analyse
(
problem
,
notifier
=
notifier
)
analyser
.
analyse
(
problem
)
basepath
=
config
.
get_basepath
()
config
.
change_basepath
(
rundir
)
...
...
@@ -659,11 +669,19 @@ def process_event(ievent, g_data_id):
# update_every=10,
# movie_filename='grond_opt_time_magnitude.mp4')
def
startThreads
():
from
.listeners
import
terminal
term
=
terminal
.
TerminalListener
(
rundir
)
term
.
start
()
return
term
term
=
startThreads
()
try
:
optimizer
=
config
.
optimizer_config
.
get_optimizer
()
if
xs_inject
is
not
None
:
from
.optimizers
import
highscore
if
not
isinstance
(
optimizer
,
highscore
.
HighScoreOptimizer
()
):
if
not
isinstance
(
optimizer
,
highscore
.
HighScoreOptimizer
):
raise
GrondError
(
'optimizer does not support injections'
)
...
...
@@ -678,6 +696,8 @@ def process_event(ievent, g_data_id):
except
BadProblem
as
e
:
logger
.
error
(
str
(
e
))
finally
:
term
.
join
()
tstop
=
time
.
time
()
logger
.
info
(
...
...
src/listeners/base.py
View file @
d36cce1c
class
Listener
(
object
):
from
threading
import
Thread
class
Listener
(
Thread
):
def
__init__
(
self
):
Thread
.
__init__
(
self
)
def
progress_start
(
self
,
name
,
niter
):
raise
NotImplementedError
()
...
...
src/listeners/terminal.py
View file @
d36cce1c
import
time
import
logging
import
numpy
as
num
from
datetime
import
timedelta
from
pyrocko
import
util
from
grond.problems
import
ModelHistory
from
.base
import
Listener
logger
=
logging
.
getLogger
(
'TerminalListener'
)
class
RingBuffer
(
num
.
ndarray
):
def
__new__
(
cls
,
*
args
,
**
kwargs
):
cls
=
num
.
ndarray
.
__new__
(
cls
,
*
args
,
**
kwargs
)
cls
.
fill
(
0.
)
return
cls
def
__init__
(
self
,
*
args
,
**
kwargs
):
self
.
pos
=
0
def
put
(
self
,
value
):
self
[
self
.
pos
]
=
value
self
.
pos
+=
1
self
.
pos
%=
self
.
size
class
color
:
PURPLE
=
'
\033
[95m'
CYAN
=
'
\033
[96m'
...
...
@@ -21,58 +45,91 @@ class TerminalListener(Listener):
row_name
=
color
.
BOLD
+
'{:<{col_param_width}s}'
+
color
.
END
parameter_fmt
=
'{:{col_width}s}'
def
__init__
(
self
):
self
.
current_state
=
None
self
.
pbars
=
{}
def
progress_start
(
self
,
name
,
niter
):
pbar
=
util
.
progressbar
(
'analysing problem'
,
niter
)
self
.
pbars
[
name
]
=
pbar
pbar
.
start
()
def
__init__
(
self
,
rundir
):
Listener
.
__init__
(
self
)
self
.
rundir
=
rundir
self
.
_iiter
=
0
self
.
iter_per_second
=
0
self
.
_iter_buffer
=
RingBuffer
(
20
)
def
run
(
self
):
logger
.
info
(
'Waiting to follow %s'
%
self
.
rundir
)
self
.
history
=
ModelHistory
.
follow
(
self
.
rundir
)
self
.
problem
=
self
.
history
.
problem
self
.
niter
=
self
.
history
.
optimizer
.
niterations
self
.
starttime
=
time
.
time
()
self
.
last_update
=
self
.
starttime
self
.
history
.
add_listener
(
self
)
self
.
start_watch
()
def
start_watch
(
self
):
while
True
:
self
.
history
.
update
()
time
.
sleep
(.
1
)
@
property
def
runtime
(
self
):
return
timedelta
(
seconds
=
time
.
time
()
-
self
.
starttime
)
@
property
def
iiter
(
self
):
return
self
.
_iiter
@
iiter
.
setter
def
iiter
(
self
,
iiter
):
dt
=
time
.
time
()
-
self
.
last_update
self
.
_iter_buffer
.
put
(
float
((
iiter
-
self
.
iiter
)
/
dt
))
self
.
iter_per_second
=
float
(
self
.
_iter_buffer
.
mean
())
self
.
_iiter
=
iiter
self
.
last_update
=
time
.
time
()
@
property
def
runtime_remaining
(
self
):
if
self
.
iter_per_second
==
0.
:
return
timedelta
()
return
timedelta
(
seconds
=
(
self
.
niter
-
self
.
iiter
)
/
self
.
iter_per_second
)
def
extend
(
self
,
*
args
):
self
.
iiter
=
self
.
history
.
nmodels
problem
=
self
.
history
.
problem
def
progress_update
(
self
,
name
,
iiter
):
self
.
pbars
[
name
].
update
(
iiter
)
def
progress_finish
(
self
,
name
):
self
.
pbars
[
name
].
finish
()
del
self
.
pbars
[
name
]
def
state
(
self
,
state
):
lines
=
[]
self
.
current_state
=
state
ladd
=
lines
.
append
def
fmt
(
s
):
return
util
.
gform
(
s
,
significant_digits
=
(
self
.
col_width
-
1
-
6
)
//
2
)
out_ln
=
self
.
row_name
+
\
''
.
join
([
self
.
parameter_fmt
]
*
len
(
state
.
parameter_sets
))
col_param_width
=
max
([
len
(
p
)
for
p
in
state
.
parameter_names
])
+
2
ladd
(
'Problem name: {s.problem_name}'
'
\t
({s.runtime.seconds} - remaining {s.runtime_remaining}'
ladd
(
'Problem name: {p.name}'
'
\t
({s.runtime} - remaining {s.runtime_remaining}'
' @ {s.iter_per_second:.1f} iter/s)'
.
format
(
s
=
s
tate
))
.
format
(
s
=
s
elf
,
p
=
problem
))
ladd
(
'Iteration {s.iiter} / {s.niter}'
.
format
(
s
=
state
))
.
format
(
s
=
self
))
out_ln
=
self
.
row_name
+
\
''
.
join
([
self
.
parameter_fmt
]
*
len
(
problem
.
parameter_names
))
col_param_width
=
max
([
len
(
p
)
for
p
in
problem
.
parameter_names
])
+
2
ladd
(
out_ln
.
format
(
*
[
'Parameter'
]
+
list
(
state
.
parameter_
sets
.
keys
()
),
*
[
'Parameter'
]
+
list
(
problem
.
parameter_
names
),
col_param_width
=
col_param_width
,
col_width
=
self
.
col_width
,
type
=
's'
))
for
ip
,
parameter_name
in
enumerate
(
state
.
parameter_names
):
ladd
(
out_ln
.
format
(
parameter_name
,
*
[
fmt
(
v
[
ip
])
for
v
in
state
.
parameter_sets
.
values
()],
col_param_width
=
col_param_width
,
col_width
=
self
.
col_width
))
#
for ip, parameter_name in enumerate(
problem
.parameter_names):
#
ladd(out_ln.format(
#
parameter_name,
#
*[fmt(v[ip]) for v in
problem
.parameter_sets.values()],
#
col_param_width=col_param_width,
#
col_width=self.col_width))
ladd
(
state
.
extra_text
.
format
(
col_param_width
=
col_param_width
,
col_width
=
self
.
col_width
,))
#
ladd(
problem
.extra_text.format(
#
col_param_width=col_param_width,
#
col_width=self.col_width,))
lines
[
0
:
0
]
=
[
'
\033
[2J'
]
ladd
(
''
)
...
...
src/meta.py
View file @
d36cce1c
...
...
@@ -184,23 +184,3 @@ class HasPaths(Object):
extra
(
op
.
normpath
(
xjoin
(
self
.
_basepath
,
xjoin
(
path_prefix
,
p
))))
for
p
in
path
]
class
Notifier
(
object
):
def
__init__
(
self
):
self
.
_listeners
=
[]
def
add_listener
(
self
,
listener
):
self
.
_listeners
.
append
(
listener
)
def
remove_listener
(
self
,
listener
):
self
.
_listeners
.
remove
(
listener
)
def
emit
(
self
,
signal_name
,
*
args
,
**
kwargs
):
for
listener
in
self
.
_listeners
:
if
not
hasattr
(
listener
,
signal_name
):
logger
.
warn
(
'signal name
\'
%s
\'
not implemented in listener %s'
%
(
signal_name
,
type
(
listener
)))
continue
getattr
(
listener
,
signal_name
)(
*
args
,
**
kwargs
)
src/optimizers/base.py
View file @
d36cce1c
import
logging
import
time
import
numpy
as
num
from
datetime
import
timedelta
from
collections
import
OrderedDict
from
pyrocko.guts
import
Object
from
..meta
import
GrondError
from
grond.meta
import
GrondError
guts_prefix
=
'grond'
...
...
@@ -17,75 +12,15 @@ class BadProblem(GrondError):
pass
class
SimpleTimedelta
(
timedelta
):
def
__str__
(
self
):
return
timedelta
.
__str__
(
self
).
split
(
'.'
)[
0
]
class
RingBuffer
(
num
.
ndarray
):
def
__new__
(
cls
,
*
args
,
**
kwargs
):
cls
=
num
.
ndarray
.
__new__
(
cls
,
*
args
,
**
kwargs
)
cls
.
fill
(
0.
)
return
cls
def
__init__
(
self
,
*
args
,
**
kwargs
):
self
.
pos
=
0
def
put
(
self
,
value
):
self
[
self
.
pos
]
=
value
self
.
pos
+=
1
self
.
pos
%=
self
.
size
class
SolverState
(
object
):
problem_name
=
''
parameter_sets
=
OrderedDict
()
parameter_names
=
[]
starttime
=
time
.
time
()
niter
=
0
iter_per_second
=
0.
extra_text
=
''
_iiter
=
0
_iter_buffer
=
RingBuffer
(
25
)
_last_update
=
time
.
time
()
@
property
def
iiter
(
self
):
return
self
.
_iiter
@
iiter
.
setter
def
iiter
(
self
,
value
):
dt
=
time
.
time
()
-
self
.
_last_update
self
.
_iter_buffer
.
put
(
float
((
value
-
self
.
_iiter
)
/
dt
))
self
.
iter_per_second
=
float
(
self
.
_iter_buffer
.
mean
())
self
.
_iiter
=
value
self
.
_last_update
=
time
.
time
()
@
property
def
runtime
(
self
):
return
timedelta
(
seconds
=
time
.
time
()
-
self
.
starttime
)
@
property
def
runtime_remaining
(
self
):
if
self
.
iter_per_second
==
0.
:
return
timedelta
()
return
timedelta
(
seconds
=
(
self
.
niter
-
self
.
iiter
)
/
self
.
iter_per_second
)
@
property
def
nparameters
(
self
):
return
len
(
self
.
parameter_names
)
class
Optimizer
(
Object
):
def
optimize
(
self
,
problem
):
raise
NotImplemented
()
@
property
def
niterations
(
self
):
raise
NotImplementedError
()
class
OptimizerConfig
(
Object
):
pass
...
...
src/optimizers/highscore/optimizer.py
View file @
d36cce1c
...
...
@@ -373,6 +373,10 @@ class HighScoreOptimizer(Optimizer):
iiter
+=
1
@
property
def
niterations
(
self
):
return
sum
([
ph
.
niterations
for
ph
in
self
.
sampler_phases
])
class
HighScoreOptimizerConfig
(
OptimizerConfig
):
...
...
src/problems/base.py
View file @
d36cce1c
...
...
@@ -4,6 +4,7 @@ import copy
import
logging
import
os.path
as
op
import
os
import
time
from
pyrocko
import
gf
,
util
,
guts
from
pyrocko.guts
import
Object
,
String
,
Bool
,
List
,
Dict
,
Int
...
...
@@ -407,11 +408,24 @@ class Problem(Object):
return
results
class
ModelHistory
(
object
):
class
InvalidRundir
(
Exception
):
pass
class
ModelHistory
(
object
):
nmodels_capacity_min
=
1024
def
__init__
(
self
,
problem
,
path
=
None
,
mode
=
'r'
):
'''Model History lets you write, read and follow new models
:param problem: The Problem
:type problem: :class:`grond.Problem`
:param path: Rundir to use, defaults to None
:type path: str, optional
:param mode: Mode to use, defaults to 'r'.
'r': Read, 'w': Write
:type mode: str, optional
'''
self
.
problem
=
problem
self
.
path
=
path
self
.
_models_buffer
=
None
...
...
@@ -421,11 +435,43 @@ class ModelHistory(object):
self
.
nmodels_capacity
=
self
.
nmodels_capacity_min
self
.
listeners
=
[]
self
.
mode
=
mode
self
.
optimizer
=
load_optimizer_info
(
path
)
if
mode
==
'r'
:
self
.
verify_rundir
(
self
.
path
)
models
,
misfits
=
load_problem_data
(
path
,
problem
)
self
.
extend
(
models
,
misfits
)
@
staticmethod
def
verify_rundir
(
rundir
):
_rundir_files
=
[
'config.yaml'
,
'optimizer.yaml'
,
'misfits'
,
'models'
]
if
not
op
.
exists
(
rundir
):
raise
OSError
(
'Rundir %s does not exist!'
%
rundir
)
for
f
in
_rundir_files
:
if
not
op
.
exists
(
op
.
join
(
rundir
,
f
)):
raise
InvalidRundir
(
'File %s not found!'
%
f
)
@
classmethod
def
follow
(
cls
,
path
,
wait
=
20.
):
'''Start following a rundir
:param path: The path to follow, a grond rundir
:type path: str, optional
:param wait: Wait time until the folder become alive, defaults to 10.
:type wait: number in seconds, optional
:returns: A ModelHistory instance
:rtype: :class:`grond.core.ModelHistory`
'''
start_watch
=
time
.
time
()
while
(
time
.
time
()
-
start_watch
)
<
wait
:
try
:
cls
.
verify_rundir
(
path
)
problem
=
load_problem_info
(
path
)
return
cls
(
problem
,
path
,
mode
=
'r'
)
except
InvalidRundir
:
time
.
sleep
(.
25
)
@
property
def
nmodels
(
self
):
if
self
.
models
is
None
:
...
...
@@ -473,7 +519,6 @@ class ModelHistory(object):
self
.
nmodels_capacity
=
self
.
nmodels_capacity_min
def
extend
(
self
,
models
,
misfits
):
nmodels
=
self
.
nmodels
n
=
models
.
shape
[
0
]
...
...
@@ -491,14 +536,13 @@ class ModelHistory(object):
self
.
misfits
=
self
.
_misfits_buffer
[:
nmodels
+
n
,
:,
:]
if
self
.
path
and
self
.
mode
==
'w'
:
for
i
in
x
range
(
n
):
for
i
in
range
(
n
):
self
.
problem
.
dump_problem_data
(
self
.
path
,
models
[
i
,
:],
misfits
[
i
,
:,
:])
self
.
emit
(
'extend'
,
nmodels
,
n
,
models
,
misfits
)
def
append
(
self
,
model
,
misfits
):
nmodels
=
self
.
nmodels
nmodels_capacity_want
=
max
(
...
...
@@ -521,18 +565,41 @@ class ModelHistory(object):
'extend'
,
nmodels
,
1
,
model
[
num
.
newaxis
,
:],
misfits
[
num
.
newaxis
,
:,
:])
def
update
(
self
):
''' Update history from path '''
nmodels_available
=
get_nmodels
(
self
.
path
,
self
.
problem
)
if
self
.
nmodels
==
nmodels_available
:
return
new_models
,
new_misfits
=
load_problem_data
(
self
.
path
,
self
.
problem
,
skip_models
=
self
.
nmodels
)
self
.
extend
(
new_models
,
new_misfits
)
def
add_listener
(
self
,
listener
):
self
.
listeners
.
append
(
listener
)
def
emit
(
self
,
event_name
,
*
args
,
**
kwargs
):
for
listener
in
self
.
listeners
:
getattr
(
listener
,
event_name
)(
*
args
,
**
kwargs
)
slot
=
getattr
(
listener
,
event_name
,
None
)
if
callable
(
slot
):
slot
(
*
args
,
**
kwargs
)
def
get_nmodels
(
dirname
,
problem
):
fn
=
op
.
join
(
dirname
,
'models'
)
with
open
(
fn
,
'r'
)
as
f
:
return
os
.
fstat
(
f
.
fileno
()).
st_size
//
(
problem
.
nparameters
*
8
)
def
load_problem_info_and_data
(
dirname
,
subset
=
None
):
problem
=
load_problem_info
(
dirname
)
xs
,
misfits
=
load_problem_data
(
xjoin
(
dirname
,
subset
),
problem
)
return
problem
,
xs
,
misfits
models
,
misfits
=
load_problem_data
(
xjoin
(
dirname
,
subset
),
problem
)
return
problem
,
models
,
misfits
def
load_optimizer_info
(
dirname
):
fn
=
op
.
join
(
dirname
,
'optimizer.yaml'
)
return
guts
.
load
(
filename
=
fn
)
def
load_problem_info
(
dirname
):
...
...
@@ -540,29 +607,35 @@ def load_problem_info(dirname):
return
guts
.
load
(
filename
=
fn
)
def
load_problem_data
(
dirname
,
problem
,
skip_models
=
0
):
def
load_problem_data
(
dirname
,
problem
,
skip_models
=
None
):
if
skip_models
is
None
:
skip_models
=
0
nmodels
=
get_nmodels
(
dirname
,
problem
)
fn
=
op
.
join
(
dirname
,
'models'
)
with
open
(
fn
,
'r'
)
as
f
:
nmodels
=
os
.
fstat
(
f
.
fileno
()).
st_size
//
(
problem
.
nparameters
*
8
)
nmodels
-=
skip_models
f
.
seek
(
skip_models
*
problem
.
nparameters
*
8
)
data1
=
num
.
fromfile
(
f
,
dtype
=
'<f8'
,
count
=
nmodels
*
problem
.
nparameters
)
\
models
=
num
.
fromfile
(
f
,
dtype
=
'<f8'
,
count
=
nmodels
*
problem
.
nparameters
)
\
.
astype
(
num
.
float
)
nmodels
=
data1
.
size
//
problem
.
nparameters
-
skip_models
xs
=
data1
.
reshape
((
nmodels
,
problem
.
nparameters
))
nmodels
=
models
.
size
//
problem
.
nparameters
models
=
models
.
reshape
((
nmodels
,
problem
.
nparameters
))
fn
=
op
.
join
(
dirname
,
'misfits'
)
with
open
(
fn
,
'r'
)
as
f
:
f
.
seek
(
skip_models
*
problem
.
ntargets
*
2
*
8
)
misfits
=
num
.
fromfile
(
f
,
dtype
=
'<f8'
,
count
=
nmodels
*
problem
.
ntargets
*
2
).
astype
(
num
.
float
)
f
,
dtype
=
'<f8'
,
count
=
nmodels
*
problem
.
ntargets
*
2
)
\
.
astype
(
num
.
float
)
misfits
=
misfits
.
reshape
((
nmodels
,
problem
.
ntargets
,
2
))
return
x
s
,
misfits
return
model
s
,
misfits
__all__
=
'''
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment