Commit 9540dd06 authored by Daniel Eggert's avatar Daniel Eggert
Browse files

refactored credentials into properties file

parent 30f22c84
...@@ -36,6 +36,7 @@ public class MetadatacrawlerModule { ...@@ -36,6 +36,7 @@ public class MetadatacrawlerModule {
private static final String INSERT_QUERY = "INSERT INTO scenes (entityid, datasetid, acquisitiondate, starttime, endtime, browseurl, dataaccessurl, downloadurl, metadataurl, modifieddate, orderurl, summary, bounds, satelliteid, sensorid, subsystemid, proc_level, cloudcover) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'METADATA', ?);"; private static final String INSERT_QUERY = "INSERT INTO scenes (entityid, datasetid, acquisitiondate, starttime, endtime, browseurl, dataaccessurl, downloadurl, metadataurl, modifieddate, orderurl, summary, bounds, satelliteid, sensorid, subsystemid, proc_level, cloudcover) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'METADATA', ?);";
@SuppressWarnings("unused")
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
Map<MetadataCrawler, List<DatasetMetadata>> crawlerTasks = new HashMap<>(); Map<MetadataCrawler, List<DatasetMetadata>> crawlerTasks = new HashMap<>();
......
...@@ -54,8 +54,8 @@ public class SyncDatabases { ...@@ -54,8 +54,8 @@ public class SyncDatabases {
connectToDestinationDB(); connectToDestinationDB();
short l8Dataset = 104; // short l8Dataset = 104;
short l7Dataset = 112; // short l7Dataset = 112;
short s2Dataset = 249; short s2Dataset = 249;
syncScenesTableForDataset(s2Dataset); syncScenesTableForDataset(s2Dataset);
} }
...@@ -110,12 +110,11 @@ public class SyncDatabases { ...@@ -110,12 +110,11 @@ public class SyncDatabases {
con.commit(); con.commit();
} }
} }
con.commit(); con.commit();
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
private static void syncScenesSentinel2Table() throws Exception { private static void syncScenesSentinel2Table() throws Exception {
// get the sceneids from the source db // get the sceneids from the source db
......
...@@ -24,14 +24,14 @@ import de.potsdam.gfz.gms.metadatacrawler.model.SceneMetadata; ...@@ -24,14 +24,14 @@ import de.potsdam.gfz.gms.metadatacrawler.model.SceneMetadata;
*/ */
public class GeoMultiSensMetadataCrawler extends MetadataCrawler { public class GeoMultiSensMetadataCrawler extends MetadataCrawler {
private static final String DB_COLUMN_LABEL_DATASET_ID = "id"; private static final String DB_COLUMN_LABEL_DATASET_ID = "id";
private static final String DB_COLUMN_LABEL_DATASET_NAME = "name"; private static final String DB_COLUMN_LABEL_DATASET_NAME = "name";
private static final String DB_COLUMN_LABEL_DATASET_DESC = "description"; private static final String DB_COLUMN_LABEL_DATASET_DESC = "description";
private static final String DB_COLUMN_LABEL_DATASET_STARTDATE = "startdate"; private static final String DB_COLUMN_LABEL_DATASET_STARTDATE = "startdate";
private static final String DB_COLUMN_LABEL_DATASET_ENDDATE = "enddate"; private static final String DB_COLUMN_LABEL_DATASET_ENDDATE = "enddate";
private static final String DB_COLUMN_LABEL_DATASET_BOUNDS = "bounds"; private static final String DB_COLUMN_LABEL_DATASET_BOUNDS = "bounds";
private static final String DB_COLUMN_LABEL_DATASET_IMAGE_TYPE = "image_type"; private static final String DB_COLUMN_LABEL_DATASET_IMAGE_TYPE = "image_type";
private static final String DB_COLUMN_LABEL_DATASET_TOTALSCENES = "totalscenes"; private static final String DB_COLUMN_LABEL_DATASET_TOTALSCENES = "totalscenes";
private static final String DB_COLUMN_LABEL_SCENE_ID = "id"; private static final String DB_COLUMN_LABEL_SCENE_ID = "id";
private static final String DB_COLUMN_LABEL_SCENE_ENTITYID = "entityid"; private static final String DB_COLUMN_LABEL_SCENE_ENTITYID = "entityid";
...@@ -56,6 +56,16 @@ public class GeoMultiSensMetadataCrawler extends MetadataCrawler { ...@@ -56,6 +56,16 @@ public class GeoMultiSensMetadataCrawler extends MetadataCrawler {
@SuppressWarnings("unused") @SuppressWarnings("unused")
private static final String DB_COLUMN_LABEL_SCENE_PROC_LEVEL = "proc_level"; private static final String DB_COLUMN_LABEL_SCENE_PROC_LEVEL = "proc_level";
/*
* (non-Javadoc)
*
* @see de.potsdam.gfz.gms.metadatacrawler.crawler.MetadataCrawler#getProviderName()
*/
@Override
protected String getProviderName() {
return "gms";
}
/* /*
* (non-Javadoc) * (non-Javadoc)
* *
......
...@@ -3,7 +3,11 @@ ...@@ -3,7 +3,11 @@
*/ */
package de.potsdam.gfz.gms.metadatacrawler.crawler; package de.potsdam.gfz.gms.metadatacrawler.crawler;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.List; import java.util.List;
import java.util.Properties;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import de.potsdam.gfz.gms.database.shared.Timestamp; import de.potsdam.gfz.gms.database.shared.Timestamp;
...@@ -17,9 +21,16 @@ import de.potsdam.gfz.gms.metadatacrawler.model.Sentinel2SceneMetadata; ...@@ -17,9 +21,16 @@ import de.potsdam.gfz.gms.metadatacrawler.model.Sentinel2SceneMetadata;
*/ */
public abstract class MetadataCrawler { public abstract class MetadataCrawler {
private boolean running = false; private static final String FILENAME_CREDENTIALS_PROPTERTIES = "credentials.properties";
private boolean canceled = false;
private CrawlTask crawlTask = null; private static final String PROPS_KEY_USERNAME = ".username";
private static final String PROPS_KEY_PASSWORD = ".password";
private static final String PROPS_DEFAULT_VALUE_USERNAME = "gms_downloader";
private static final String PROPS_DEFAULT_VALUE_PASSWORD = "gms_downloader";
private boolean running = false;
private boolean canceled = false;
private CrawlTask crawlTask = null;
private class CrawlTask implements Runnable { private class CrawlTask implements Runnable {
...@@ -227,4 +238,55 @@ public abstract class MetadataCrawler { ...@@ -227,4 +238,55 @@ public abstract class MetadataCrawler {
*/ */
protected abstract void crawlScenes(DatasetMetadata dataset, Timestamp start, Timestamp end, BlockingQueue<SceneMetadata> resultQueue); protected abstract void crawlScenes(DatasetMetadata dataset, Timestamp start, Timestamp end, BlockingQueue<SceneMetadata> resultQueue);
protected abstract String getProviderName();
/**
* @return
*/
private static Properties loadCredentials() {
Properties props = new Properties();
try {
InputStream propsStream = MetadataCrawler.class.getResourceAsStream("/" + FILENAME_CREDENTIALS_PROPTERTIES);
if (propsStream == null) {
// not found on classpath - check root folder
File propsFile = new File(FILENAME_CREDENTIALS_PROPTERTIES);
if (!propsFile.exists()) {
System.err.println(FILENAME_CREDENTIALS_PROPTERTIES + " neither found on classpath nor on current directory. ");
return props;
}
propsStream = new FileInputStream(propsFile);
}
props.load(propsStream);
} catch (Throwable t) {
System.err.println("Error parsing Propertiesfile: " + FILENAME_CREDENTIALS_PROPTERTIES + "\t " + t.getMessage());
}
return props;
}
protected String[] getCredentials() {
Properties props = loadCredentials();
String[] credentials = new String[] { PROPS_DEFAULT_VALUE_USERNAME, PROPS_DEFAULT_VALUE_PASSWORD };
String provider = getProviderName();
String keyUsername = provider + PROPS_KEY_USERNAME;
String keyPassword = provider + PROPS_KEY_PASSWORD;
if (props.containsKey(keyUsername)) {
credentials[0] = props.getProperty(keyUsername, PROPS_DEFAULT_VALUE_USERNAME);
}
if (props.containsKey(keyPassword)) {
credentials[1] = props.getProperty(keyPassword, PROPS_DEFAULT_VALUE_PASSWORD);
}
return credentials;
}
} }
...@@ -22,7 +22,6 @@ import de.potsdam.gfz.gms.database.shared.Timestamp; ...@@ -22,7 +22,6 @@ import de.potsdam.gfz.gms.database.shared.Timestamp;
import de.potsdam.gfz.gms.metadatacrawler.model.DatasetMetadata; import de.potsdam.gfz.gms.metadatacrawler.model.DatasetMetadata;
import de.potsdam.gfz.gms.metadatacrawler.model.SceneMetadata; import de.potsdam.gfz.gms.metadatacrawler.model.SceneMetadata;
import de.potsdam.gfz.gms.metadatacrawler.model.Sentinel2SceneMetadata; import de.potsdam.gfz.gms.metadatacrawler.model.Sentinel2SceneMetadata;
import de.potsdam.gfz.scihubapi.SciHubCredentials;
import de.potsdam.gfz.scihubapi.opensearch.SciHubSearch; import de.potsdam.gfz.scihubapi.opensearch.SciHubSearch;
import de.potsdam.gfz.scihubapi.opensearch.SearchQuery; import de.potsdam.gfz.scihubapi.opensearch.SearchQuery;
import de.potsdam.gfz.scihubapi.opensearch.SearchResultEntry; import de.potsdam.gfz.scihubapi.opensearch.SearchResultEntry;
...@@ -35,7 +34,7 @@ public class ScientificDataHubCrawler extends MetadataCrawler { ...@@ -35,7 +34,7 @@ public class ScientificDataHubCrawler extends MetadataCrawler {
// private static final Logger LOG = LoggerFactory.getLogger(ScientificDataHubCrawler.class); // private static final Logger LOG = LoggerFactory.getLogger(ScientificDataHubCrawler.class);
private final SciHubSearch sciHubSearchApi = new SciHubSearch(SciHubCredentials.USERNAME, SciHubCredentials.PASSWORD); private final SciHubSearch sciHubSearchApi;
// private static final String ENTRY_FIELD_NAME_ID = "id"; // private static final String ENTRY_FIELD_NAME_ID = "id";
// private static final String ENTRY_FIELD_NAME_IDENTIFIER = "identifier"; // private static final String ENTRY_FIELD_NAME_IDENTIFIER = "identifier";
...@@ -47,6 +46,24 @@ public class ScientificDataHubCrawler extends MetadataCrawler { ...@@ -47,6 +46,24 @@ public class ScientificDataHubCrawler extends MetadataCrawler {
// private static final String user = "gmsdb"; // private static final String user = "gmsdb";
// private static final String password = "gmsdb"; // private static final String password = "gmsdb";
/**
*
*/
public ScientificDataHubCrawler() {
String[] credentials = getCredentials();
sciHubSearchApi = new SciHubSearch(credentials[0], credentials[1]);
}
/*
* (non-Javadoc)
*
* @see de.potsdam.gfz.gms.metadatacrawler.crawler.MetadataCrawler#getProviderName()
*/
@Override
protected String getProviderName() {
return "scihub";
}
@Override @Override
protected void crawlScenes(DatasetMetadata dataset, Timestamp start, Timestamp end, BlockingQueue<SceneMetadata> resultQueue) { protected void crawlScenes(DatasetMetadata dataset, Timestamp start, Timestamp end, BlockingQueue<SceneMetadata> resultQueue) {
while (sciHubSearchApi.isRunning()) { while (sciHubSearchApi.isRunning()) {
......
...@@ -16,7 +16,6 @@ import org.postgis.Polygon; ...@@ -16,7 +16,6 @@ import org.postgis.Polygon;
import de.potsdam.gfz.gms.database.shared.Timestamp; import de.potsdam.gfz.gms.database.shared.Timestamp;
import de.potsdam.gfz.gms.metadatacrawler.model.DatasetMetadata; import de.potsdam.gfz.gms.metadatacrawler.model.DatasetMetadata;
import de.potsdam.gfz.gms.metadatacrawler.model.SceneMetadata; import de.potsdam.gfz.gms.metadatacrawler.model.SceneMetadata;
import de.potsdam.gfz.usgsapi.UsgsCredentials;
import de.potsdam.gfz.usgsapi.json.ErrorCode; import de.potsdam.gfz.usgsapi.json.ErrorCode;
import de.potsdam.gfz.usgsapi.json.UsgsApi; import de.potsdam.gfz.usgsapi.json.UsgsApi;
import de.potsdam.gfz.usgsapi.json.datamodels.Bounds; import de.potsdam.gfz.usgsapi.json.datamodels.Bounds;
...@@ -38,12 +37,12 @@ import de.potsdam.gfz.usgsapi.json.response.SceneSearchResponse; ...@@ -38,12 +37,12 @@ import de.potsdam.gfz.usgsapi.json.response.SceneSearchResponse;
*/ */
public class UsgsCrawler extends MetadataCrawler { public class UsgsCrawler extends MetadataCrawler {
private String apiKey = null; private String apiKey = null;
private long apiKeyExpireTime = 0l; private long apiKeyExpireTime = 0l;
private static final long APIKEY_EXPIRE_GRACE = 5 * 60 * 1000; // 5 min private static final long APIKEY_EXPIRE_GRACE = 5 * 60 * 1000; // 5 min
private static final int MAX_RESULTS_PER_QUERY = 1000; private static final int MAX_RESULTS_PER_QUERY = 1000;
/* /*
* (non-Javadoc) * (non-Javadoc)
...@@ -233,6 +232,16 @@ public class UsgsCrawler extends MetadataCrawler { ...@@ -233,6 +232,16 @@ public class UsgsCrawler extends MetadataCrawler {
return true; return true;
} }
/*
* (non-Javadoc)
*
* @see de.potsdam.gfz.gms.metadatacrawler.crawler.MetadataCrawler#getProviderName()
*/
@Override
protected String getProviderName() {
return "usgs";
}
/** /**
* Returns <code>true</code> in we are still logged in or the login attempt was successful, <code>false</code> otherwise * Returns <code>true</code> in we are still logged in or the login attempt was successful, <code>false</code> otherwise
* *
...@@ -253,7 +262,9 @@ public class UsgsCrawler extends MetadataCrawler { ...@@ -253,7 +262,9 @@ public class UsgsCrawler extends MetadataCrawler {
boolean error = false; boolean error = false;
LoginResponse response = null; LoginResponse response = null;
try { try {
response = UsgsApi.sendRequest(new LoginRequest(UsgsCredentials.USERNAME, UsgsCredentials.PASSWORD)); String[] credentials = getCredentials();
response = UsgsApi.sendRequest(new LoginRequest(credentials[0], credentials[1]));
error = (response == null || response.hasError()); error = (response == null || response.hasError());
} catch (Exception e) { } catch (Exception e) {
System.err.println(e.getMessage()); System.err.println(e.getMessage());
...@@ -278,7 +289,7 @@ public class UsgsCrawler extends MetadataCrawler { ...@@ -278,7 +289,7 @@ public class UsgsCrawler extends MetadataCrawler {
if (!ensureLogin()) { if (!ensureLogin()) {
// unable to login - return empty list // unable to login - return empty list
System.err.println("login error"); System.err.println("login error");
return new ArrayList<DatasetMetadata>(0); return new ArrayList<>(0);
} }
try { try {
...@@ -312,7 +323,7 @@ public class UsgsCrawler extends MetadataCrawler { ...@@ -312,7 +323,7 @@ public class UsgsCrawler extends MetadataCrawler {
return datasets; return datasets;
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
return new ArrayList<DatasetMetadata>(0); return new ArrayList<>(0);
} }
} }
......
# usgs credentials
usgs.username = eggert
usgs.password = |R8o.Yeop3#
# nasa credentials
nasa.username = gms_downloader
nasa.password = AJOucCitedAcce0
# esa/scihub credentials
scihub.username = gms_downloader
scihub.password = gms_downloader
\ No newline at end of file
...@@ -19,7 +19,6 @@ import org.postgis.PGgeometry; ...@@ -19,7 +19,6 @@ import org.postgis.PGgeometry;
import de.potsdam.gfz.gms.database.misc.DatabaseConfigConstants; import de.potsdam.gfz.gms.database.misc.DatabaseConfigConstants;
import de.potsdam.gfz.gms.database.shared.Timestamp; import de.potsdam.gfz.gms.database.shared.Timestamp;
import de.potsdam.gfz.gms.database.shared.Timestamp.TimeUnits; import de.potsdam.gfz.gms.database.shared.Timestamp.TimeUnits;
import de.potsdam.gfz.scihubapi.SciHubCredentials;
import de.potsdam.gfz.scihubapi.opendata.SciHubOpenData; import de.potsdam.gfz.scihubapi.opendata.SciHubOpenData;
/** /**
...@@ -64,7 +63,10 @@ public class InsertS2GranulesIntoSceneDB { ...@@ -64,7 +63,10 @@ public class InsertS2GranulesIntoSceneDB {
private static final String url = "jdbc:postgresql://" + host + "/geomultisens"; private static final String url = "jdbc:postgresql://" + host + "/geomultisens";
private static final String user = "gmsdb"; private static final String user = "gmsdb";
private static final String password = "gmsdb"; private static final String password = "gmsdb";
private static final SciHubOpenData sciHubData = new SciHubOpenData(SciHubCredentials.USERNAME, SciHubCredentials.PASSWORD); private static final SciHubOpenData sciHubData = new SciHubOpenData("username", "password"); // TODO:
// set
// actual
// credentials
private static final Random RAND = new Random(); private static final Random RAND = new Random();
......
...@@ -7,7 +7,6 @@ import java.util.concurrent.BlockingQueue; ...@@ -7,7 +7,6 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
import de.potsdam.gfz.gms.database.shared.Timestamp; import de.potsdam.gfz.gms.database.shared.Timestamp;
import de.potsdam.gfz.scihubapi.SciHubCredentials;
import de.potsdam.gfz.scihubapi.opensearch.SciHubSearch; import de.potsdam.gfz.scihubapi.opensearch.SciHubSearch;
import de.potsdam.gfz.scihubapi.opensearch.SearchQuery; import de.potsdam.gfz.scihubapi.opensearch.SearchQuery;
import de.potsdam.gfz.scihubapi.opensearch.SearchResultEntry; import de.potsdam.gfz.scihubapi.opensearch.SearchResultEntry;
...@@ -16,12 +15,12 @@ public class ScihubApiTester { ...@@ -16,12 +15,12 @@ public class ScihubApiTester {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
// init api // init api
SciHubSearch scihubSearch = new SciHubSearch(SciHubCredentials.USERNAME, SciHubCredentials.PASSWORD); SciHubSearch scihubSearch = new SciHubSearch("username", "password");
// define search query // define search query
SearchQuery query = new SearchQuery(); SearchQuery query = new SearchQuery();
query.addParameter(SearchQuery.KEYWORD_BEGINPOSITION, "[ " + new Timestamp(2016, 9, 27, 6, 40).toISO8601String() query.addParameter(SearchQuery.KEYWORD_BEGINPOSITION,
+ " TO " + new Timestamp(2016, 9, 27, 7,58).toISO8601String() + "]"); "[ " + new Timestamp(2016, 9, 27, 6, 40).toISO8601String() + " TO " + new Timestamp(2016, 9, 27, 7, 58).toISO8601String() + "]");
query.addParameter(SearchQuery.KEYWORD_PLATFORMNAME, "Sentinel-2"); query.addParameter(SearchQuery.KEYWORD_PLATFORMNAME, "Sentinel-2");
// query.addParameter(SearchQuery.KEYWORD_FOOTPRINT, "\"Intersects(52.407703, 12.159007)\""); // query.addParameter(SearchQuery.KEYWORD_FOOTPRINT, "\"Intersects(52.407703, 12.159007)\"");
...@@ -41,24 +40,24 @@ public class ScihubApiTester { ...@@ -41,24 +40,24 @@ public class ScihubApiTester {
public int compare(SearchResultEntry o1, SearchResultEntry o2) { public int compare(SearchResultEntry o1, SearchResultEntry o2) {
Instant t1 = (Instant) o1.getFieldValue("beginposition"); Instant t1 = (Instant) o1.getFieldValue("beginposition");
Instant t2 = (Instant) o2.getFieldValue("beginposition"); Instant t2 = (Instant) o2.getFieldValue("beginposition");
if (t1.isBefore(t2) ) { if (t1.isBefore(t2)) {
return -1; return -1;
} else if(t1.isAfter(t2)) { } else if (t1.isAfter(t2)) {
return 1; return 1;
} else { } else {
return 0; return 0;
} }
} }
}); });
for (SearchResultEntry entry : resultList) { for (SearchResultEntry entry : resultList) {
// printFields(entry); // printFields(entry);
System.out.println(entry.getFieldValue("id") + " | " + entry.getFieldValue("beginposition") + " | " System.out.println(entry.getFieldValue("id") + " | " + entry.getFieldValue("beginposition") + " | " + entry.getFieldValue("size"));
+ entry.getFieldValue("size"));
} }
} }
@SuppressWarnings("unused")
private static void printFields(SearchResultEntry entry) { private static void printFields(SearchResultEntry entry) {
for (String field : entry.getFields()) { for (String field : entry.getFields()) {
System.out.println(field + ":\t" + entry.getFieldValue(field)); System.out.println(field + ":\t" + entry.getFieldValue(field));
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment