refactored credentials into properties file

gms-metadatacrawler/src/main/java/de/potsdam/gfz/gms/metadatacrawler/MetadatacrawlerModule.java

@@ -36,6 +36,7 @@ public class MetadatacrawlerModule {
 
 	private static final String INSERT_QUERY = "INSERT INTO scenes (entityid, datasetid, acquisitiondate, starttime, endtime, browseurl, dataaccessurl, downloadurl, metadataurl, modifieddate, orderurl, summary, bounds, satelliteid, sensorid, subsystemid, proc_level, cloudcover) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'METADATA', ?);";
 
+	@SuppressWarnings("unused")
 	public static void main(String[] args) throws Exception {
 
 		Map<MetadataCrawler, List<DatasetMetadata>> crawlerTasks = new HashMap<>();

gms-metadatacrawler/src/main/java/de/potsdam/gfz/gms/metadatacrawler/SyncDatabases.java

@@ -54,8 +54,8 @@ public class SyncDatabases {
 
 		connectToDestinationDB();
 
-		short l8Dataset = 104;
-		short l7Dataset = 112;
+		// short l8Dataset = 104;
+		// short l7Dataset = 112;
 		short s2Dataset = 249;
 		syncScenesTableForDataset(s2Dataset);
 	}
@@ -110,12 +110,11 @@ public class SyncDatabases {
 				con.commit();
 			}
 		}
-		
+
 		con.commit();
 
 	}
 
-	
 	@SuppressWarnings("unused")
 	private static void syncScenesSentinel2Table() throws Exception {
 		// get the sceneids from the source db

gms-metadatacrawler/src/main/java/de/potsdam/gfz/gms/metadatacrawler/crawler/GeoMultiSensMetadataCrawler.java

@@ -24,14 +24,14 @@ import de.potsdam.gfz.gms.metadatacrawler.model.SceneMetadata;
  */
 public class GeoMultiSensMetadataCrawler extends MetadataCrawler {
 
-	private static final String	DB_COLUMN_LABEL_DATASET_ID			= "id";
-	private static final String	DB_COLUMN_LABEL_DATASET_NAME		= "name";
-	private static final String	DB_COLUMN_LABEL_DATASET_DESC		= "description";
-	private static final String	DB_COLUMN_LABEL_DATASET_STARTDATE	= "startdate";
-	private static final String	DB_COLUMN_LABEL_DATASET_ENDDATE		= "enddate";
-	private static final String	DB_COLUMN_LABEL_DATASET_BOUNDS		= "bounds";
-	private static final String	DB_COLUMN_LABEL_DATASET_IMAGE_TYPE	= "image_type";
-	private static final String	DB_COLUMN_LABEL_DATASET_TOTALSCENES	= "totalscenes";
+	private static final String	DB_COLUMN_LABEL_DATASET_ID				= "id";
+	private static final String	DB_COLUMN_LABEL_DATASET_NAME			= "name";
+	private static final String	DB_COLUMN_LABEL_DATASET_DESC			= "description";
+	private static final String	DB_COLUMN_LABEL_DATASET_STARTDATE		= "startdate";
+	private static final String	DB_COLUMN_LABEL_DATASET_ENDDATE			= "enddate";
+	private static final String	DB_COLUMN_LABEL_DATASET_BOUNDS			= "bounds";
+	private static final String	DB_COLUMN_LABEL_DATASET_IMAGE_TYPE		= "image_type";
+	private static final String	DB_COLUMN_LABEL_DATASET_TOTALSCENES		= "totalscenes";
 
 	private static final String	DB_COLUMN_LABEL_SCENE_ID				= "id";
 	private static final String	DB_COLUMN_LABEL_SCENE_ENTITYID			= "entityid";
@@ -56,6 +56,16 @@ public class GeoMultiSensMetadataCrawler extends MetadataCrawler {
 	@SuppressWarnings("unused")
 	private static final String	DB_COLUMN_LABEL_SCENE_PROC_LEVEL		= "proc_level";
 
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see de.potsdam.gfz.gms.metadatacrawler.crawler.MetadataCrawler#getProviderName()
+	 */
+	@Override
+	protected String getProviderName() {
+		return "gms";
+	}
+
 	/*
 	 * (non-Javadoc)
 	 * 

gms-metadatacrawler/src/main/java/de/potsdam/gfz/gms/metadatacrawler/crawler/MetadataCrawler.java

@@ -3,7 +3,11 @@
  */
 package de.potsdam.gfz.gms.metadatacrawler.crawler;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
 import java.util.List;
+import java.util.Properties;
 import java.util.concurrent.BlockingQueue;
 
 import de.potsdam.gfz.gms.database.shared.Timestamp;
@@ -17,9 +21,16 @@ import de.potsdam.gfz.gms.metadatacrawler.model.Sentinel2SceneMetadata;
  */
 public abstract class MetadataCrawler {
 
-	private boolean		running		= false;
-	private boolean		canceled	= false;
-	private CrawlTask	crawlTask	= null;
+	private static final String	FILENAME_CREDENTIALS_PROPTERTIES	= "credentials.properties";
+
+	private static final String	PROPS_KEY_USERNAME					= ".username";
+	private static final String	PROPS_KEY_PASSWORD					= ".password";
+	private static final String	PROPS_DEFAULT_VALUE_USERNAME		= "gms_downloader";
+	private static final String	PROPS_DEFAULT_VALUE_PASSWORD		= "gms_downloader";
+
+	private boolean				running								= false;
+	private boolean				canceled							= false;
+	private CrawlTask			crawlTask							= null;
 
 	private class CrawlTask implements Runnable {
 
@@ -227,4 +238,55 @@ public abstract class MetadataCrawler {
 	 */
 	protected abstract void crawlScenes(DatasetMetadata dataset, Timestamp start, Timestamp end, BlockingQueue<SceneMetadata> resultQueue);
 
+	protected abstract String getProviderName();
+
+	/**
+	 * @return
+	 */
+	private static Properties loadCredentials() {
+		Properties props = new Properties();
+
+		try {
+
+			InputStream propsStream = MetadataCrawler.class.getResourceAsStream("/" + FILENAME_CREDENTIALS_PROPTERTIES);
+
+			if (propsStream == null) {
+				// not found on classpath - check root folder
+				File propsFile = new File(FILENAME_CREDENTIALS_PROPTERTIES);
+				if (!propsFile.exists()) {
+					System.err.println(FILENAME_CREDENTIALS_PROPTERTIES + " neither found on classpath nor on current directory. ");
+					return props;
+				}
+
+				propsStream = new FileInputStream(propsFile);
+			}
+
+			props.load(propsStream);
+		} catch (Throwable t) {
+			System.err.println("Error parsing Propertiesfile: " + FILENAME_CREDENTIALS_PROPTERTIES + "\t " + t.getMessage());
+		}
+
+		return props;
+	}
+
+	protected String[] getCredentials() {
+		Properties props = loadCredentials();
+
+		String[] credentials = new String[] { PROPS_DEFAULT_VALUE_USERNAME, PROPS_DEFAULT_VALUE_PASSWORD };
+
+		String provider = getProviderName();
+		String keyUsername = provider + PROPS_KEY_USERNAME;
+		String keyPassword = provider + PROPS_KEY_PASSWORD;
+
+		if (props.containsKey(keyUsername)) {
+			credentials[0] = props.getProperty(keyUsername, PROPS_DEFAULT_VALUE_USERNAME);
+		}
+
+		if (props.containsKey(keyPassword)) {
+			credentials[1] = props.getProperty(keyPassword, PROPS_DEFAULT_VALUE_PASSWORD);
+		}
+
+		return credentials;
+	}
+
 }

gms-metadatacrawler/src/main/java/de/potsdam/gfz/gms/metadatacrawler/crawler/ScientificDataHubCrawler.java

@@ -22,7 +22,6 @@ import de.potsdam.gfz.gms.database.shared.Timestamp;
 import de.potsdam.gfz.gms.metadatacrawler.model.DatasetMetadata;
 import de.potsdam.gfz.gms.metadatacrawler.model.SceneMetadata;
 import de.potsdam.gfz.gms.metadatacrawler.model.Sentinel2SceneMetadata;
-import de.potsdam.gfz.scihubapi.SciHubCredentials;
 import de.potsdam.gfz.scihubapi.opensearch.SciHubSearch;
 import de.potsdam.gfz.scihubapi.opensearch.SearchQuery;
 import de.potsdam.gfz.scihubapi.opensearch.SearchResultEntry;
@@ -35,7 +34,7 @@ public class ScientificDataHubCrawler extends MetadataCrawler {
 
 	// private static final Logger LOG = LoggerFactory.getLogger(ScientificDataHubCrawler.class);
 
-	private final SciHubSearch sciHubSearchApi = new SciHubSearch(SciHubCredentials.USERNAME, SciHubCredentials.PASSWORD);
+	private final SciHubSearch sciHubSearchApi;
 
 	// private static final String ENTRY_FIELD_NAME_ID = "id";
 	// private static final String ENTRY_FIELD_NAME_IDENTIFIER = "identifier";
@@ -47,6 +46,24 @@ public class ScientificDataHubCrawler extends MetadataCrawler {
 	// private static final String user = "gmsdb";
 	// private static final String password = "gmsdb";
 
+	/**
+	 * 
+	 */
+	public ScientificDataHubCrawler() {
+		String[] credentials = getCredentials();
+		sciHubSearchApi = new SciHubSearch(credentials[0], credentials[1]);
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see de.potsdam.gfz.gms.metadatacrawler.crawler.MetadataCrawler#getProviderName()
+	 */
+	@Override
+	protected String getProviderName() {
+		return "scihub";
+	}
+
 	@Override
 	protected void crawlScenes(DatasetMetadata dataset, Timestamp start, Timestamp end, BlockingQueue<SceneMetadata> resultQueue) {
 		while (sciHubSearchApi.isRunning()) {

gms-metadatacrawler/src/main/java/de/potsdam/gfz/gms/metadatacrawler/crawler/UsgsCrawler.java

@@ -16,7 +16,6 @@ import org.postgis.Polygon;
 import de.potsdam.gfz.gms.database.shared.Timestamp;
 import de.potsdam.gfz.gms.metadatacrawler.model.DatasetMetadata;
 import de.potsdam.gfz.gms.metadatacrawler.model.SceneMetadata;
-import de.potsdam.gfz.usgsapi.UsgsCredentials;
 import de.potsdam.gfz.usgsapi.json.ErrorCode;
 import de.potsdam.gfz.usgsapi.json.UsgsApi;
 import de.potsdam.gfz.usgsapi.json.datamodels.Bounds;
@@ -38,12 +37,12 @@ import de.potsdam.gfz.usgsapi.json.response.SceneSearchResponse;
  */
 public class UsgsCrawler extends MetadataCrawler {
 
-	private String	apiKey				= null;
-	private long	apiKeyExpireTime	= 0l;
+	private String				apiKey					= null;
+	private long				apiKeyExpireTime		= 0l;
 
-	private static final long APIKEY_EXPIRE_GRACE = 5 * 60 * 1000;	// 5 min
+	private static final long	APIKEY_EXPIRE_GRACE		= 5 * 60 * 1000;	// 5 min
 
-	private static final int MAX_RESULTS_PER_QUERY = 1000;
+	private static final int	MAX_RESULTS_PER_QUERY	= 1000;
 
 	/*
 	 * (non-Javadoc)
@@ -233,6 +232,16 @@ public class UsgsCrawler extends MetadataCrawler {
 		return true;
 	}
 
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see de.potsdam.gfz.gms.metadatacrawler.crawler.MetadataCrawler#getProviderName()
+	 */
+	@Override
+	protected String getProviderName() {
+		return "usgs";
+	}
+
 	/**
 	 * Returns <code>true</code> in we are still logged in or the login attempt was successful, <code>false</code> otherwise
 	 * 
@@ -253,7 +262,9 @@ public class UsgsCrawler extends MetadataCrawler {
 		boolean error = false;
 		LoginResponse response = null;
 		try {
-			response = UsgsApi.sendRequest(new LoginRequest(UsgsCredentials.USERNAME, UsgsCredentials.PASSWORD));
+			String[] credentials = getCredentials();
+
+			response = UsgsApi.sendRequest(new LoginRequest(credentials[0], credentials[1]));
 			error = (response == null || response.hasError());
 		} catch (Exception e) {
 			System.err.println(e.getMessage());
@@ -278,7 +289,7 @@ public class UsgsCrawler extends MetadataCrawler {
 		if (!ensureLogin()) {
 			// unable to login - return empty list
 			System.err.println("login error");
-			return new ArrayList<DatasetMetadata>(0);
+			return new ArrayList<>(0);
 		}
 
 		try {
@@ -312,7 +323,7 @@ public class UsgsCrawler extends MetadataCrawler {
 			return datasets;
 		} catch (Exception e) {
 			e.printStackTrace();
-			return new ArrayList<DatasetMetadata>(0);
+			return new ArrayList<>(0);
 		}
 	}
 

gms-metadatacrawler/src/main/resources/credentials.properties

+# usgs credentials
+usgs.username = eggert
+usgs.password = |R8o.Yeop3#
+
+# nasa credentials
+nasa.username = gms_downloader
+nasa.password = AJOucCitedAcce0
+
+# esa/scihub credentials
+scihub.username = gms_downloader
+scihub.password = gms_downloader
\ No newline at end of file

gms-metadatacrawler/src/test/java/InsertS2GranulesIntoSceneDB.java

@@ -19,7 +19,6 @@ import org.postgis.PGgeometry;
 import de.potsdam.gfz.gms.database.misc.DatabaseConfigConstants;
 import de.potsdam.gfz.gms.database.shared.Timestamp;
 import de.potsdam.gfz.gms.database.shared.Timestamp.TimeUnits;
-import de.potsdam.gfz.scihubapi.SciHubCredentials;
 import de.potsdam.gfz.scihubapi.opendata.SciHubOpenData;
 
 /**
@@ -64,7 +63,10 @@ public class InsertS2GranulesIntoSceneDB {
 	private static final String									url							= "jdbc:postgresql://" + host + "/geomultisens";
 	private static final String									user						= "gmsdb";
 	private static final String									password					= "gmsdb";
-	private static final SciHubOpenData							sciHubData					= new SciHubOpenData(SciHubCredentials.USERNAME, SciHubCredentials.PASSWORD);
+	private static final SciHubOpenData							sciHubData					= new SciHubOpenData("username", "password");																																																																												 // TODO:
+																																																																																																														 // set
+																																																																																																														 // actual
+																																																																																																														 // credentials
 
 	private static final Random									RAND						= new Random();
 

gms-metadatacrawler/src/test/java/ScihubApiTester.java

@@ -7,7 +7,6 @@ import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
 import de.potsdam.gfz.gms.database.shared.Timestamp;
-import de.potsdam.gfz.scihubapi.SciHubCredentials;
 import de.potsdam.gfz.scihubapi.opensearch.SciHubSearch;
 import de.potsdam.gfz.scihubapi.opensearch.SearchQuery;
 import de.potsdam.gfz.scihubapi.opensearch.SearchResultEntry;
@@ -16,12 +15,12 @@ public class ScihubApiTester {
 
 	public static void main(String[] args) throws Exception {
 		// init api
-		SciHubSearch scihubSearch = new SciHubSearch(SciHubCredentials.USERNAME, SciHubCredentials.PASSWORD);
+		SciHubSearch scihubSearch = new SciHubSearch("username", "password");
 
 		// define search query
 		SearchQuery query = new SearchQuery();
-		query.addParameter(SearchQuery.KEYWORD_BEGINPOSITION, "[ " + new Timestamp(2016, 9, 27, 6, 40).toISO8601String()
-				+ " TO " + new Timestamp(2016, 9, 27, 7,58).toISO8601String() + "]");
+		query.addParameter(SearchQuery.KEYWORD_BEGINPOSITION,
+				"[ " + new Timestamp(2016, 9, 27, 6, 40).toISO8601String() + " TO " + new Timestamp(2016, 9, 27, 7, 58).toISO8601String() + "]");
 		query.addParameter(SearchQuery.KEYWORD_PLATFORMNAME, "Sentinel-2");
 		// query.addParameter(SearchQuery.KEYWORD_FOOTPRINT, "\"Intersects(52.407703, 12.159007)\"");
 
@@ -41,24 +40,24 @@ public class ScihubApiTester {
 			public int compare(SearchResultEntry o1, SearchResultEntry o2) {
 				Instant t1 = (Instant) o1.getFieldValue("beginposition");
 				Instant t2 = (Instant) o2.getFieldValue("beginposition");
-				if (t1.isBefore(t2) ) {
+				if (t1.isBefore(t2)) {
 					return -1;
-				} else if(t1.isAfter(t2)) {
+				} else if (t1.isAfter(t2)) {
 					return 1;
 				} else {
 					return 0;
 				}
 			}
 		});
-		
+
 		for (SearchResultEntry entry : resultList) {
 			// printFields(entry);
-			System.out.println(entry.getFieldValue("id") + " | " + entry.getFieldValue("beginposition") + " | "
-					+ entry.getFieldValue("size"));
+			System.out.println(entry.getFieldValue("id") + " | " + entry.getFieldValue("beginposition") + " | " + entry.getFieldValue("size"));
 		}
 
 	}
 
+	@SuppressWarnings("unused")
 	private static void printFields(SearchResultEntry entry) {
 		for (String field : entry.getFields()) {
 			System.out.println(field + ":\t" + entry.getFieldValue(field));