Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
geomultisens
gms_preprocessing
Commits
d1e17d47
Commit
d1e17d47
authored
Dec 11, 2017
by
Daniel Scheffler
Browse files
Added some not yet working code.
parent
a8c36936
Changes
1
Hide whitespace changes
Inline
Side-by-side
gms_preprocessing/algorithms/L2B_P.py
View file @
d1e17d47
...
...
@@ -19,6 +19,8 @@ from glob import glob
import
re
import
json
from
collections
import
OrderedDict
import
dill
import
itertools
from
sklearn.cluster
import
k_means_
# noqa F401 # flake8 issue
from
sklearn.model_selection
import
train_test_split
...
...
@@ -366,29 +368,83 @@ class _MachineLearner_RSImage(object):
self
.
train_X
,
self
.
test_X
,
self
.
train_Y
,
self
.
test_Y
=
\
train_test_split
(
self
.
spectra_X
,
self
.
spectra_Y
,
test_size
=
test_size
,
shuffle
=
True
,
random_state
=
0
)
@
classmethod
def
from_dump
(
cls
,
dillFile
):
# type: (str) -> _MachineLearner_RSImage
with
open
(
dillFile
,
'rb'
)
as
inF
:
ML_instance
=
dill
.
load
(
inF
)
if
not
isinstance
(
ML_instance
,
cls
):
raise
ValueError
(
'The given dillFile does not contain an instance of %s but %s.'
%
(
cls
.
__name__
,
type
(
ML_instance
)))
return
ML_instance
def
__getstate__
(
self
):
def
dump
(
self
,
path_out
,
exclude_arrays
=
True
):
if
exclude_arrays
:
instance2dump
=
copy
()
with
open
(
path_out
,
'wb'
)
as
outF
:
dill
.
dump
(
self
,
outF
)
@
staticmethod
def
_im2spectra
(
geoArr
):
"""Convert images to array of spectra samples (rows: samples; cols: spectral information)."""
return
geoArr
.
reshape
((
geoArr
.
rows
*
geoArr
.
cols
,
geoArr
.
bands
))
def
plot_scatter_matrix
(
self
,
figsize
=
(
15
,
15
)):
@
staticmethod
def
_spectra2im
(
spectra
,
rows
,
cols
):
"""Convert array of spectra samples (rows: samples; cols: spectral information) to a 3D image."""
return
spectra
.
reshape
(
rows
,
cols
,
spectra
.
shape
[
1
])
def
plot_scatter_matrix
(
self
,
figsize
=
(
15
,
15
),
mode
=
'intersensor'
):
train_X
=
self
.
train_X
[
np
.
random
.
choice
(
self
.
train_X
.
shape
[
0
],
1000
,
replace
=
False
),
:]
train_Y
=
self
.
train_Y
[
np
.
random
.
choice
(
self
.
train_Y
.
shape
[
0
],
1000
,
replace
=
False
),
:]
df
=
DataFrame
(
train_X
,
columns
=
[
'Band %s'
%
b
for
b
in
range
(
1
,
self
.
im_X
.
bands
+
1
)])
scatter_matrix
(
df
,
figsize
=
figsize
,
marker
=
'.'
,
hist_kwds
=
{
'bins'
:
50
},
s
=
30
,
alpha
=
0.8
)
plt
.
suptitle
(
'Image X band to band correlation'
)
if
mode
==
'intersensor'
:
import
seaborn
fig
,
axes
=
plt
.
subplots
(
train_X
.
shape
[
1
],
train_Y
.
shape
[
1
],
figsize
=
(
25
,
9
),
sharex
=
'all'
,
sharey
=
'all'
)
#fig.suptitle('Correlation of %s and %s bands' % (self.src_cube.satellite, self.tgt_cube.satellite), size=25)
color
=
seaborn
.
hls_palette
(
13
)
for
i
,
ax
in
zip
(
range
(
train_X
.
shape
[
1
]),
axes
.
flatten
()):
for
j
,
ax
in
zip
(
range
(
train_Y
.
shape
[
1
]),
axes
.
flatten
()):
axes
[
i
,
j
].
scatter
(
train_X
[:,
j
],
train_Y
[:,
i
],
c
=
color
[
j
],
label
=
str
(
j
))
#axes[i, j].set_xlim(-0.1, 1.1)
#axes[i, j].set_ylim(-0.1, 1.1)
# if j == 8:
# axes[5, j].set_xlabel('S2 B8A\n' + str(metadata_s2['Bands_S2'][j]) + ' nm', size=10)
# elif j in range(9, 13):
# axes[5, j].set_xlabel('S2 B' + str(j) + '\n' + str(metadata_s2['Bands_S2'][j]) + ' nm', size=10)
# else:
# axes[5, j].set_xlabel('S2 B' + str(j + 1) + '\n' + str(metadata_s2['Bands_S2'][j]) + ' nm',
# size=10)
# axes[i, 0].set_ylabel(
# 'S3 SLSTR B' + str(6 - i) + '\n' + str(metadata_s3['Bands_S3'][5 - i]) + ' nm',
# size=10)
#axes[4, j].set_xticks(np.arange(0, 1.2, 0.2))
#axes[i, j].plot([0, 1], [0, 1], c='red')
df
=
DataFrame
(
train_Y
,
columns
=
[
'Band %s'
%
b
for
b
in
range
(
1
,
self
.
im_Y
.
bands
+
1
)])
scatter_matrix
(
df
,
figsize
=
figsize
,
marker
=
'.'
,
hist_kwds
=
{
'bins'
:
50
},
s
=
30
,
alpha
=
0.8
)
plt
.
suptitle
(
'Image Y band to band correlation'
)
else
:
df
=
DataFrame
(
train_X
,
columns
=
[
'Band %s'
%
b
for
b
in
range
(
1
,
self
.
im_X
.
bands
+
1
)])
scatter_matrix
(
df
,
figsize
=
figsize
,
marker
=
'.'
,
hist_kwds
=
{
'bins'
:
50
},
s
=
30
,
alpha
=
0.8
)
plt
.
suptitle
(
'Image X band to band correlation'
)
df
=
DataFrame
(
train_Y
,
columns
=
[
'Band %s'
%
b
for
b
in
range
(
1
,
self
.
im_Y
.
bands
+
1
)])
scatter_matrix
(
df
,
figsize
=
figsize
,
marker
=
'.'
,
hist_kwds
=
{
'bins'
:
50
},
s
=
30
,
alpha
=
0.8
)
plt
.
suptitle
(
'Image Y band to band correlation'
)
class
LinearRegression_RSImage
(
_MachineLearner_RSImage
):
def
__init__
(
self
,
im_X
,
im_Y
,
test_size
=
0.6
):
super
(
LinearRegression_RSImage
,
self
).
__init__
(
im_X
,
im_Y
,
test_size
=
test_size
)
self
.
linRegressor
=
LinearRegression
().
fit
(
self
.
train_X
,
self
.
train_Y
)
self
.
linRegressor
=
LinearRegression
().
fit
(
self
.
train_X
,
self
.
train_Y
)
# type: LinearRegression
@
property
def
coefficients_
(
self
):
...
...
@@ -408,6 +464,25 @@ class LinearRegression_RSImage(_MachineLearner_RSImage):
return
dict
(
train
=
self
.
linRegressor
.
score
(
self
.
train_X
,
self
.
train_Y
),
test
=
self
.
linRegressor
.
score
(
self
.
test_X
,
self
.
test_Y
))
def
predict
(
self
,
im_Y
,
nodataVal
=
None
):
# type: (Union[GeoArray, np.ndarray]) -> np.ndarray
# 3D -> 2D
spectra_Y
=
self
.
_im2spectra
(
im_Y
)
# predict
spectra_predicted
=
self
.
linRegressor
.
predict
(
spectra_Y
).
astype
(
im_Y
.
dtype
)
# 2D -> 3D
image_predicted
=
self
.
_spectra2im
(
spectra_predicted
,
rows
=
im_Y
.
shape
[
0
],
cols
=
im_Y
.
shape
[
1
])
# re-apply nodata values to predicted result
if
nodataVal
is
not
None
:
im_Y_gA
=
GeoArray
(
im_Y
,
nodata
=
nodataVal
)
image_predicted
[
im_Y_gA
.
mask_nodata
[:]
==
0
]
=
nodataVal
return
image_predicted
class
RidgeRegression_RSImage
(
_MachineLearner_RSImage
):
def
__init__
(
self
,
im_X
,
im_Y
,
test_size
=
0.6
):
...
...
@@ -886,17 +961,71 @@ class RefCube(object):
class
Classifier_Trainer
(
object
):
def
__init__
(
self
,
src_sensor_refcube
,
tgt_sensor_refcube
):
# type: (RefCube, RefCube) -> None
# type: (
Union[
RefCube,
str], Union[
RefCube
, str]
) -> None
"""
:param src_sensor_refcube: file path of reference cube of source sensor
:param tgt_sensor_refcube: file path of reference cube of target sensor
:param src_sensor_refcube: file path of reference cube of source sensor
or instance of RefCube
:param tgt_sensor_refcube: file path of reference cube of target sensor
or instance of RefCube
"""
self
.
src_cube
=
src_sensor_refcube
self
.
tgt_cube
=
tgt_sensor_refcube
self
.
src_cube
=
src_sensor_refcube
if
isinstance
(
src_sensor_refcube
,
RefCube
)
else
RefCube
(
src_sensor_refcube
)
self
.
tgt_cube
=
tgt_sensor_refcube
if
isinstance
(
src_sensor_refcube
,
RefCube
)
else
RefCube
(
src_sensor_refcube
)
def
plot_scattermatrix
(
self
):
pass
import
seaborn
fig
,
axes
=
plt
.
subplots
(
self
.
src_cube
.
data
.
bands
,
self
.
tgt_cube
.
data
.
bands
,
figsize
=
(
25
,
9
),
sharex
=
'all'
,
sharey
=
'all'
)
fig
.
suptitle
(
'Correlation of %s and %s bands'
%
(
self
.
src_cube
.
satellite
,
self
.
tgt_cube
.
satellite
),
size
=
25
)
color
=
seaborn
.
hls_palette
(
13
)
for
i
,
ax
in
zip
(
range
(
6
),
axes
.
flatten
()):
for
j
,
ax
in
zip
(
range
(
13
),
axes
.
flatten
()):
axes
[
i
,
j
].
scatter
(
train_s2
[:,
j
],
train_s3
[:,
5
-
i
],
c
=
color
[
j
],
label
=
str
(
j
))
axes
[
i
,
j
].
set_xlim
(
-
0.1
,
1.1
)
axes
[
i
,
j
].
set_ylim
(
-
0.1
,
1.1
)
if
j
==
8
:
axes
[
5
,
j
].
set_xlabel
(
'S2 B8A
\n
'
+
str
(
metadata_s2
[
'Bands_S2'
][
j
])
+
' nm'
,
size
=
10
)
elif
j
in
range
(
9
,
13
):
axes
[
5
,
j
].
set_xlabel
(
'S2 B'
+
str
(
j
)
+
'
\n
'
+
str
(
metadata_s2
[
'Bands_S2'
][
j
])
+
' nm'
,
size
=
10
)
else
:
axes
[
5
,
j
].
set_xlabel
(
'S2 B'
+
str
(
j
+
1
)
+
'
\n
'
+
str
(
metadata_s2
[
'Bands_S2'
][
j
])
+
' nm'
,
size
=
10
)
axes
[
i
,
0
].
set_ylabel
(
'S3 SLSTR B'
+
str
(
6
-
i
)
+
'
\n
'
+
str
(
metadata_s3
[
'Bands_S3'
][
5
-
i
])
+
' nm'
,
size
=
10
)
axes
[
4
,
j
].
set_xticks
(
np
.
arange
(
0
,
1.2
,
0.2
))
axes
[
i
,
j
].
plot
([
0
,
1
],
[
0
,
1
],
c
=
'red'
)
def
show_band_scatterplot
(
self
,
band_src_im
,
band_tgt_im
):
from
scipy.stats
import
gaussian_kde
x
=
self
.
src_cube
.
data
[
band_src_im
].
flatten
()[:
10000
]
y
=
self
.
tgt_cube
.
data
[
band_tgt_im
].
flatten
()[:
10000
]
# Calculate the point density
xy
=
np
.
vstack
([
x
,
y
])
z
=
gaussian_kde
(
xy
)(
xy
)
fig
=
plt
.
figure
(
figsize
=
(
15
,
15
))
plt
.
scatter
(
x
,
y
,
c
=
z
,
s
=
30
,
edgecolor
=
''
)
plt
.
show
()
class
Classifier_Generator
(
object
):
def
__init__
(
self
,
list_refcubes
):
# type: (List[Union[str, RefCube]]) -> None
self
.
refcubes
=
[
RefCube
(
inRC
)
if
isinstance
(
inRC
,
str
)
else
inRC
for
inRC
in
list_refcubes
]
def
create_classifiers
(
self
,
outDir
,
clsApproach
=
'LR'
,
*
args
,
**
kwargs
):
for
src_cube
,
tgt_cube
in
itertools
.
combinations
(
self
.
refcubes
,
r
=
2
):
# type: RefCube
print
(
src_cube
.
satellite
,
src_cube
.
sensor
,
tgt_cube
.
satellite
,
tgt_cube
.
sensor
)
fName_cls
=
'__'
.
join
([
clsApproach
,
src_cube
.
satellite
,
src_cube
.
sensor
])
+
\
'__to__'
+
'__'
.
join
([
tgt_cube
.
satellite
,
tgt_cube
.
sensor
])
+
'.dill'
print
(
"Creating classifier %s..."
%
fName_cls
)
ML
=
specHomoApproaches
[
clsApproach
](
src_cube
.
data
,
tgt_cube
.
data
)
ML
.
dump
(
os
.
path
.
join
(
outDir
,
fName_cls
))
class
SpecHomo_Classifier
(
object
):
...
...
@@ -905,3 +1034,9 @@ class SpecHomo_Classifier(object):
def
from_refcubes
(
self
,
dir_refcubes
):
pass
specHomoApproaches
=
dict
(
LR
=
LinearRegression_RSImage
,
RR
=
RidgeRegression_RSImage
)
Daniel Scheffler
@danschef
mentioned in commit
3523b7e4
·
Feb 06, 2018
mentioned in commit
3523b7e4
mentioned in commit 3523b7e45dfa1c9251c552bedcfa965d43f5e384
Toggle commit list
Daniel Scheffler
@danschef
mentioned in commit
f017740d
·
Mar 31, 2020
mentioned in commit
f017740d
mentioned in commit f017740dc0d88d8fd0c2f90698bc4651a04b1521
Toggle commit list
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment