added support for LANDSAT_8_C1 dataset

gms-metadatacrawler/.gitignore

+/failed.txt

gms-metadatacrawler/src/main/java/de/potsdam/gfz/gms/metadatacrawler/CloudCoverUpdate.java

@@ -47,7 +47,7 @@ public class CloudCoverUpdate {
 
 	public static void main(String[] args) throws Exception {
 		// landsat_tm(108), landsat_mss(107), landsat_8(104), landsat_7(112)
-		short datasetids[] = new short[] { 104, 112, 189 }; // , 108, 107, 112 };
+		short datasetids[] = new short[] { 189 }; // , 108, 107, 112 };
 		// Timestamp start = Timestamp.MIN;
 		// Timestamp end = Timestamp.now();
 		// Timestamp end = start.plus(20, Timestamp.TimeUnits.DAYS);

gms-metadatacrawler/src/main/java/de/potsdam/gfz/gms/metadatacrawler/MetadatacrawlerModule.java

@@ -39,29 +39,34 @@ public class MetadatacrawlerModule {
 	public static void main(String[] args) throws Exception {
 
 		Map<MetadataCrawler, List<DatasetMetadata>> crawlerTasks = new HashMap<>();
-
-		// crawl landsat 7 and 8 from usgs
+		//
+		// // crawl landsat 7 and 8 from usgs
 		MetadataCrawler usgs = new UsgsCrawler();
+		// for(DatasetMetadata dm : usgs.getAvailableDatasets(new Timestamp(2017), new Timestamp(2018))) {
+		// if(dm.name.equals("LANDSAT_8_C1")) {
+		// System.out.println(dm.name + " \t " + dm.description + " \t " + dm.imageType + " \t ");
+		// }
+		// }
 		// printProvidedDatasets(usgs);
 
-		DatasetMetadata landsat7 = new DatasetMetadata();
-		landsat7.id = 112;
-		landsat7.name = "LANDSAT_ETM_SLC_OFF";
-		DatasetMetadata landsat8 = new DatasetMetadata();
-		landsat8.id = 104;
-		landsat8.name = "LANDSAT_8";
-		DatasetMetadata aster_l1t = new DatasetMetadata();
-		aster_l1t.id = 189;
-		aster_l1t.name = "ASTER_L1T";
-		// crawlerTasks.put(usgs, Arrays.asList(landsat7, landsat8));
-		crawlerTasks.put(usgs, Arrays.asList(aster_l1t));
+		// DatasetMetadata landsat7 = new DatasetMetadata();
+		// landsat7.id = 112;
+		// landsat7.name = "LANDSAT_ETM_SLC_OFF";
+		DatasetMetadata landsat8c1 = new DatasetMetadata();
+		landsat8c1.id = 250;
+		landsat8c1.name = "LANDSAT_8_C1";
+		// DatasetMetadata aster_l1t = new DatasetMetadata();
+		// aster_l1t.id = 189;
+		// aster_l1t.name = "ASTER_L1T";
+		crawlerTasks.put(usgs, Arrays.asList(landsat8c1));
+		// crawlerTasks.put(usgs, Arrays.asList(aster_l1t, landsat7, landsat8));
 
 		// crawl sentinel2 from esa' scientific data hub
 		MetadataCrawler scihub = new ScientificDataHubCrawler();
 		DatasetMetadata s2 = new DatasetMetadata();
 		s2.id = 249;
 		s2.name = "Sentinel-2";
-		crawlerTasks.put(scihub, Arrays.asList(s2));
+		// crawlerTasks.put(scihub, Arrays.asList(s2));
 
 		final boolean queryLatestOnly = true;
 
@@ -72,7 +77,7 @@ public class MetadatacrawlerModule {
 
 			// crawl datasets
 			for (DatasetMetadata dataset : datasets) {
-				crawlDataset(dataset, crawler, queryLatestOnly, null);
+				crawlDataset(dataset, crawler, queryLatestOnly, null);// new Timestamp(2017, 5, 1));
 			}
 		}
 	}
@@ -129,12 +134,11 @@ public class MetadatacrawlerModule {
 		BlockingQueue<SceneMetadata> queue = new LinkedBlockingQueue<>();
 
 		Timestamp start = null;
-		Timestamp end = null;
+		Timestamp end = Timestamp.now().plus(1, Timestamp.TimeUnits.DAYS);
 
 		// there are scenes query later ones
 		if (queryLatestOnly && !latestDateInDB.equals(Timestamp.MIN)) {
 			start = latestDateInDB;
-			end = Timestamp.now().plus(1, Timestamp.TimeUnits.DAYS);
 		}
 
 		if (startOverride != null) {
@@ -168,6 +172,7 @@ public class MetadatacrawlerModule {
 
 			if (scene.satelliteid == 0) {
 				crawler.cancelCrawling();
+				dbScenes.clear();
 				throw new IllegalArgumentException("The scenes have to be tagged with the satelliteid - update crawler class \"" + crawler.getClass().getSimpleName()
 						+ "\" to support crawling of dataset \"" + dataset.name + "\"");
 			}
@@ -210,21 +215,36 @@ public class MetadatacrawlerModule {
 			}
 		}
 
+		dbScenes.clear();
+
 		System.out.println("inserted " + count + " scenes into database");
 		db.commit();
 
 		// update dataset count
-		ResultSet rs = db.placeCustomQuery("SELECT count(id) from scenes where datasetid = " + dataset.id + ";");
-		int totalScenes = 0;
+		ResultSet rs = db.placeCustomQuery("SELECT count(id), min(acquisitiondate), max(acquisitiondate) from scenes where datasetid = " + dataset.id + ";");
+		long totalScenes = 0;
+		java.sql.Timestamp maxDate = null;
+		java.sql.Timestamp minDate = null;
+
 		if (rs.next()) {
-			totalScenes = rs.getInt(1);
-			db.prepareCustomStatement("UPDATE datasets SET totalscenes = " + totalScenes + " where id = " + dataset.id + ";").executeUpdate();
+			totalScenes = rs.getLong(1);
+			minDate = rs.getTimestamp(2);
+			maxDate = rs.getTimestamp(3);
+
+			PreparedStatement dsUpPst = db.prepareCustomStatement("UPDATE datasets SET totalscenes = ?, startdate = ?, enddate = ? where id = ?;");
+			dsUpPst.setLong(1, totalScenes);
+			dsUpPst.setTimestamp(2, minDate);
+			dsUpPst.setTimestamp(3, maxDate);
+			dsUpPst.setShort(4, dataset.id);
+			dsUpPst.executeUpdate();
 			db.commit();
+
 		}
 
 		System.out.println("\ndataset: " + dataset.name);
 		System.out.println("crawler: " + crawler.getClass().getSimpleName());
 		System.out.println("total scenes in DB: " + totalScenes);
+		System.out.println("date range in DB: " + minDate.toString() + " TO " + maxDate.toString());
 		System.out.println("********** Crawling finished **********\n");
 	}
 

gms-metadatacrawler/src/main/java/de/potsdam/gfz/gms/metadatacrawler/crawler/MetadataCrawler.java

@@ -102,8 +102,9 @@ public abstract class MetadataCrawler {
 			scene.sensorid = 9;	// ASTER
 			scene.subsystemid = 0; // null
 			break;
+		case 250:
 		case 104:
-			// LANDSAT_8
+			// LANDSAT_8 and LANDSAT_8_C1
 			scene.satelliteid = 7;	// Landsat-8
 			scene.subsystemid = 0;	// null