L8 bulk update

gms-metadatacrawler/src/main/java/de/potsdam/gfz/gms/metadatacrawler/LandsatCollection1CsvUpdate.java

 package de.potsdam.gfz.gms.metadatacrawler;
 
 import java.nio.file.Paths;
+import java.sql.PreparedStatement;
 import java.sql.ResultSet;
-import java.util.HashSet;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Scanner;
-import java.util.Set;
 
 import de.potsdam.gfz.gms.database.SceneDatabase;
 
 public class LandsatCollection1CsvUpdate {
 
-	// browseAvailable,browseURL,sceneID,LANDSAT_PRODUCT_ID,sensor,acquisitionDate,dateUpdated,path,row,upperLeftCornerLatitude,upperLeftCornerLongitude,upperRightCornerLatitude,upperRightCornerLongitude,lowerLeftCornerLatitude,lowerLeftCornerLongitude,lowerRightCornerLatitude,lowerRightCornerLongitude,sceneCenterLatitude,sceneCenterLongitude,cloudCover,cloudCoverFull,dayOrNight,sunElevation,sunAzimuth,receivingStation,sceneStartTime,sceneStopTime,imageQuality1,DATA_TYPE_L1,cartURL,ROLL_ANGLE,GEOMETRIC_RMSE_MODEL,GEOMETRIC_RMSE_MODEL_X,GEOMETRIC_RMSE_MODEL_Y,FULL_PARTIAL_SCENE,NADIR_OFFNADIR,PROCESSING_SOFTWARE_VERSION,CPF_NAME,RLUT_FILE_NAME,BPF_NAME_OLI,BPF_NAME_TIRS,GROUND_CONTROL_POINTS_MODEL,GROUND_CONTROL_POINTS_VERSION,DATE_L1_GENERATED,TIRS_SSM_MODEL,COLLECTION_NUMBER,COLLECTION_CATEGORY,CLOUD_COVER_LAND
+	private static class Scene {
+		long id;
+		String entityId;
+
+		Scene(long id, String entityid) {
+			this.id = id;
+			this.entityId = entityid;
+		}
+	}
 
+	// browseAvailable,browseURL,sceneID,LANDSAT_PRODUCT_ID,sensor,acquisitionDate,dateUpdated,path,row,upperLeftCornerLatitude,upperLeftCornerLongitude,upperRightCornerLatitude,upperRightCornerLongitude,lowerLeftCornerLatitude,lowerLeftCornerLongitude,lowerRightCornerLatitude,lowerRightCornerLongitude,sceneCenterLatitude,sceneCenterLongitude,cloudCover,cloudCoverFull,dayOrNight,sunElevation,sunAzimuth,receivingStation,sceneStartTime,sceneStopTime,imageQuality1,DATA_TYPE_L1,cartURL,ROLL_ANGLE,GEOMETRIC_RMSE_MODEL,GEOMETRIC_RMSE_MODEL_X,GEOMETRIC_RMSE_MODEL_Y,FULL_PARTIAL_SCENE,NADIR_OFFNADIR,PROCESSING_SOFTWARE_VERSION,CPF_NAME,RLUT_FILE_NAME,BPF_NAME_OLI,BPF_NAME_TIRS,GROUND_CONTROL_POINTS_MODEL,GROUND_CONTROL_POINTS_VERSION,DATE_L1_GENERATED,TIRS_SSM_MODEL,COLLECTION_NUMBER,COLLECTION_CATEGORY,CLOUD_COVER_LAND
 	public static void main(String[] args) throws Throwable {
 		// get all entityids for L8C1 from database as well as from the csv file
 		SceneDatabase db = SceneDatabase.getInstance();
 
-		ResultSet rs = db.placeCustomQuery("select entityid from scenes where datasetid=250;");
+		PreparedStatement updatePst = db.prepareCustomStatement("update scenes set entityid=? where id=?;");
+
+		ResultSet rs = db.placeCustomQuery(
+				"select id, entityid from scenes where datasetid=250 and proc_level<'DOWNLOADED'::proc_level;");
 
-		Set<String> dbEntityIds = new HashSet<>();
+		Map<String, Scene> dbEntityIds = new HashMap<>();
 
 		while (rs.next()) {
-			String entityid = rs.getString(1);
-			dbEntityIds.add(entityid.substring(0, entityid.length() - 2));
+			long id = rs.getLong(1);
+			String entityid = rs.getString(2);
+			dbEntityIds.put(entityid.substring(0, entityid.length() - 2), new Scene(id, entityid));
 		}
 
 		System.out.println(dbEntityIds.size() + " scenes in db");
 
-		// Set<String> csvEntityIds = new HashSet<>();
-
-		int matchCount = 0;
-		int noMatchCount = 0;
-
 		Scanner s = new Scanner(Paths.get("/home/eggert/Downloads/LANDSAT_8_C1.csv"));
 
 		// skip header
@@ -41,24 +50,31 @@ public class LandsatCollection1CsvUpdate {
 			String[] lineSplit = s.nextLine().split(",");
 
 			String entityid = lineSplit[2];
-
-			entityid = entityid.substring(0, entityid.length() - 2);
-
-			if (dbEntityIds.contains(entityid)) {
-				// match found
-				dbEntityIds.remove(entityid);
-				++matchCount;
-			} else {
-				// csvEntityIds.add(entityid);
-				++noMatchCount;
+			String mapKey = entityid.substring(0, entityid.length() - 2);
+
+			if (dbEntityIds.containsKey(mapKey)) {
+				// match found - check entire entityId
+				Scene scene = dbEntityIds.remove(mapKey);
+				if (!scene.entityId.equals(entityid)) {
+					// entityids differ in last two digits - update db
+					updatePst.setLong(2, scene.id);
+					updatePst.setString(1, entityid);
+					updatePst.executeUpdate();
+					// db.commit();
+				}
+
+				if (dbEntityIds.size() % 1000 == 0) {
+					db.commit();
+					System.out.println(dbEntityIds.size() + " left.");
+				}
 			}
 		}
 
 		s.close();
 
+		db.commit();
 		System.out.println(dbEntityIds.size() + " unmatched scenes in db");
-		System.out.println(noMatchCount + " unmatched scenes in csv");
-		System.out.println(matchCount + " matching scenes in db and csv");
+
 	}
 
 }