diff --git a/README.md b/README.md index a458f7e668a13c92a9c749c2d354dfdef812825c..f76ce41ca37d3500a6be7b2eb0a32e4542e123c9 100755 --- a/README.md +++ b/README.md @@ -7,6 +7,9 @@ Status [Coverage report](http://gts2.gitext.gfz-potsdam.de/gts2_client/coverage/) ## Release notes +* **2018-02-16:** Added installation information and installation script for miniconda and all needed packages +* **2018-01-18:** Added docker file and script for building an compatible image and running a container for the client +* **2018-01-17:** Added option for mosaicing/merging tifs and RGBs on client side * **2018-01-10:** Added output of RGB images into jpg or png, nc-output still in progress ## Description @@ -16,8 +19,8 @@ time of interest and wanted band combination and saves them to geotiff (.tiff) or alternatively as .json file or as netcdf (.nc) file. ## Requirements -1. Access to GTS2 API (username and password) -1. Python 3 +1. Access to GTS2 API (username, password, port) +1. Python3 1. Python packages: * numpy * gdal @@ -33,21 +36,55 @@ Clone the repository with: ## Installation -1. Make sure your system meets all the Requirements (see above) - * For GFZ users: you can achieve that by using the gfz python. - For that please run: module load pygfz -2. Install the package by first going into the repository folder and then run: - python gts2_client/setup.py install -3. Create a credentials file in your home directory **credentials_gts2_client** - that contains the following structure: +### ... is easy (as long your python is compatible): +* Make sure your system meets all the requirements (see above) +* Install the package by running: +`python gts2_client/setup.py install` + +### Install a compatible python: +The following instruction is only valid for Linux distributions but can be adapted on Windows machines. +Everything is based on bash shell. + +#### EITHER: Use install script and follow instructions: +Run: `bash install_py_gts2_client.sh` + +After providing the installation path the script will install miniconda +and all necessary packages as well as the gts2_client. +At the script will provide instructions which PATHs to add to your .bashrc. + +#### OR: Per hand (expert modus): +* Download and install at least Miniconda with default settings: +``` +wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh +chmod 755 Miniconda3-latest-Linux-x86_64.sh +./Miniconda3-latest-Linux-x86_64.sh -b +rm -f Miniconda3-latest-Linux-x86_64.sh +``` +* Install necessary packages for gts2_client using **gts2_client_conda_install.yml**: +`conda env update -f gts2_client_conda_install.yml` + + +### Finally: +Create a credentials file in your home directory **credentials_gts2_client** +that contains the following structure: ```bash {"user": "", -"password": ""} +"password": "" +"port": ""} ``` Please fill in your credentials (provided by GFZ Potsdam, please contact gts2@gfz-potsdam.de). +### If you want to use docker +**!! Attention: Expert mode !! root access to computer is needed.** +You only have to run **docker_gts2_client/build_run_gts2_client_docker.sh** +It creates an image that is based on a Centos:7, performs all needed installation steps and starts a container. +You end up with a shell where you can run `gts2_client`. +The building of the image can take some time (up to 30 Minutes), but once it is done, +**docker_gts2_client/build_run_gts2_client_docker.sh** skips the build and only starts the container. + + ## Usage ### As command line tool: @@ -56,7 +93,7 @@ gts2_client.py required_arguments [optional_arguments] ``` The list of arguments including their default values can be called from the command line with: -gts2_client --help +`gts2_client --help` #### Arguments: ##### Required: @@ -108,8 +145,14 @@ The list of arguments including their default values can be called from the comm * -q RGB_BANDS_SELECTION, --rgb_bands_selection RGB_BANDS_SELECTION band selection for rgb production, choose from: realistic, nice_looking, vegetation, - healthy_vegetation_urban, water, snow, agriculture + healthy_vegetation_urban, snow, agriculture (default: realistic) + * -w MERGE_TIFS, --merge_tifs MERGE_TIFS + Merge tifs and RGBs if area in two or more MGRS tiles + per time step (True or False). (default: False) + * -x MERGE_TILE, --merge_tile MERGE_TILE + Choose MGRS tile into which the merge of files is + performed (e.g. 33UUV). (default: None) ### As python package @@ -122,3 +165,6 @@ result = gts2_client.client(out_mode="python", **kwargs) ## Limitations: * Bands with spatial resolution 60m (Band 1) are not stacked. + * Requests with areas larger than 0.2°x0.2° will probably not be processed + if you also request a large time range this threshold can also be smaller + diff --git a/credentials_gts2_client b/credentials_gts2_client index f5dd342783f7b292ace2d0a6a72e657945dd6021..d1721f38623a1f07da66d7383c18679ab149f8c7 100644 --- a/credentials_gts2_client +++ b/credentials_gts2_client @@ -1,4 +1,5 @@ { "user": "", - "password": "" + "password": "", + "port": 80 } \ No newline at end of file diff --git a/docker_gts2_client/build_run_gts2_client_docker.sh b/docker_gts2_client/build_run_gts2_client_docker.sh new file mode 100755 index 0000000000000000000000000000000000000000..3214ccdc45ba9941822cb1ac71b3d223bdd3ff08 --- /dev/null +++ b/docker_gts2_client/build_run_gts2_client_docker.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +context_dir="./context" +dockerfile="gts2_client.docker" +runner_os="centos" +runner_iname="gts2_client_runner" +runner_tag="${runner_os}:${runner_iname}" +container_name="gts2_client" +cred_file="$HOME/credentials_gts2_client" +out_data_folder="/tmp/gts2_client" + +#Check if image exists +if [ $(sudo docker images | grep ${runner_iname} | wc -l) == 0 ] +then + + # Copying credentials file into context dir, if exists + if [ -e ${cred_file} ] + then + echo "cp ${cred_file} ./context/" + cp ${cred_file} ./context/ + else + echo "read" + read -p "Please enter GTS2 username: " username + read -p "Please enter GTS2 password: " password + read -p "Please enter GTS2 port (default 80): " port + cat > ${cred_file} <= thres) | (latrange >= thres): + raise ValueError("Your requestst area too large: ({lon:7.5f}°x{lat:7.5f}° exceeds {thres}°x{thres}°)" + "".format(lat=latrange, lon=lonrange, thres=thres)) + + def __init__(self, opts, logger=None): + + self.check_geo(ll=opts["ll"], ur=opts["ur"]) + + address = "https://rz-vm175.gfz-potsdam.de:{port}/AC".format(port=opts["auth"]["port"]) # test parameters such that api string formatting wont fail - assert len(ur) == 2 - assert len(ll) == 2 - assert sensor in ["S2A", "S2all"] - assert level in ["L1C", "L2A"] - - # define api pattern - if level == "L1C": - self.api_fmt = "".join([ - "{address}/{bands}/{date_from}_{date_to}/{ll_lon:.5f}_{ur_lon:.5f}_{ll_lat:.5f}_{ur_lat:.5f}", - "?sensor={sensor}&level={level}&minimum_fill={minimum_fill}&utm_zone={utm_zone}"]) - else: - self.api_fmt = "".join([ - "{address}/{bands}/{date_from}_{date_to}/{ll_lon:.5f}_{ur_lon:.5f}_{ll_lat:.5f}_{ur_lat:.5f}", - "?minimum_fill={minimum_fill}&sensor={sensor}&level={level}&version={version}&suffix={suffix}&" - "max_cloudy={max_cloudy}&utm_zone={utm_zone}" - ]) + assert len(opts["ur"]) == 2 + assert len(opts["ll"]) == 2 + assert opts["sensor"] in ["S2A", "S2B", "S2all"] + assert opts["level"] in ["L1C", "L2A"] + + self.api_fmt = "{address}/{bands}/{date_from}_{date_to}/{ll_lon:.5f}_{ur_lon:.5f}_{ll_lat:.5f}_{ur_lat:.5f}" \ + "?minimum_fill={minimum_fill}&sensor={sensor}&level={level}&version={version}&suffix={suffix}" \ + "&max_cloudy={max_cloudy}&utm_zone={utm_zone}" + # construct api call - self.api_call = self.api_fmt.format(address=address, bands=bands, date_from=date_from, date_to=date_to, - suffix=suffix, - minimum_fill=minimum_fill, sensor=sensor, level=level, version=version, - ll_lon=ll[0], ll_lat=ll[1], - ur_lon=ur[0], ur_lat=ur[1], max_cloudy=max_cloudy, utm_zone=utm_zone) + self.api_call = self.api_fmt.format(address=address, bands=opts["bands"], date_from=opts["date_from"], + date_to=opts["date_to"], suffix=opts["suffix"], level=opts["level"], + minimum_fill=opts["minimum_fill"], sensor=opts["sensor"], + version=opts["version"], max_cloudy=opts["max_cloudy"], + ll_lon=opts["ll"][0], ll_lat=opts["ll"][1], + ur_lon=opts["ur"][0], ur_lat=opts["ur"][1], utm_zone=opts["utm_zone"]) logger.info(">>>>>> API-call: %s " % self.api_call) # get data, update dict from json - self.update(requests.get(self.api_call, verify=False, auth=auth).json()) + result = requests.get(self.api_call, verify=False, auth=opts["auth"]["auth"]) + status = {"http_code": result.status_code} + + self.update(result.json()) + self.update(status) + + +def __spawn(cmd, cwd): + """ + Runs shell command (cmd) in the working directory (cwd). + :param cmd: string + :param cwd: string + :return: + """ + pp = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, cwd=cwd) + stdout, stderr = pp.communicate() + if pp.returncode != 0: + raise ValueError( + "cmd:%s,rc:%s,stderr:%s,stdout:%s" % (cmd, str(pp.returncode), str(stderr), str(stdout))) + + +def __merge_tifs_same_timestamp(tifs, outpath, target_tile=""): + """ + Transforms all tifs from timestep to same utm zon (from target tile) and merges them together. + :param outpath: string + :param tifs: List of strings or None + :param target_tile: 5-element string + :return: + """ + basepath = os.path.dirname(tifs[0]) + utm = target_tile[0:2] + base_split = os.path.basename(tifs[0]).split("_") + merge_file = "{path}/{pref}_{tile}_{suff}".format(path=outpath, pref="_".join(base_split[0:4]), tile=target_tile, + suff="_".join(base_split[5::])) + + with tempfile.TemporaryDirectory(dir=basepath) as tmp_dir: + for fi in tifs: + if target_tile not in fi: + outfile = "{tmp_dir}/{orig_fn}_{tile}.tif".format(orig_fn=os.path.basename(fi.split(".")[0]), + tile=target_tile, tmp_dir=tmp_dir) + cmd = "gdalwarp -t_srs '+proj=utm +zone={utm} +datum=WGS84' -overwrite {infile} {outfile} ".format( + infile=fi, outfile=outfile, utm=utm) + __spawn(cmd, tmp_dir) + + else: + shutil.copy(fi, tmp_dir) + + merge_list = " ".join(glob(os.path.join(tmp_dir, "*.tif"))) + cmd = "gdal_merge.py -o {merge_file} {merge_list}".format(merge_file=merge_file, merge_list=merge_list) + __spawn(cmd, tmp_dir) + + return merge_file + + +def __find_double_timestamp(files): + """ + Finds filenames with doublicate time stamp (but different MGRS tile) in list of files. + :param files: list of strings + :return: dict + """ + + basepath = os.path.dirname(files[0]) + prefix = "_".join(os.path.basename(files[0]).split("_")[0:3]) + + bands = list(set(["_".join(os.path.basename(fa).split("_")[5::]).split(".")[0] for fa in files])) + + double_dict = {} + for bi in bands: + band_files = [s for s in files if "_{band}.".format(band=bi) in s] + dates = [os.path.basename(bf).split("_")[3] for bf in band_files] + double_dates = ([item for item, count in collections.Counter(dates).items() if count > 1]) + + double_files = {} + for di in double_dates: + double_files[di] = glob(os.path.join(basepath, "{pref}*_{date}*{band}.*".format( + band=bi, date=di, pref=prefix))) + + double_dict[bi] = double_files + + return double_dict + + +def merge_tiles(tif_list, out_mode="single", target_tile=None): + """ + Performs merge of tifs of the same timestep that are devided into two different MGRS tiles + :param tif_list: list of strings + :param out_mode: string + :param target_tile: 5-element string or None + :return: + """ + + stack = True if out_mode == "stack" else False + + double_dict = __find_double_timestamp(tif_list) + for bi in double_dict.keys(): + for dates, tifs_one_date in double_dict[bi].items(): + basepath = os.path.dirname(tifs_one_date[0]) + if len(tifs_one_date) > 1: + + if target_tile is None: + tiles = [re.search('[0-9]{2,2}[A-Z]{3,3}', ti).group(0).replace("_", "") for ti in tifs_one_date] + target_tile = list(set(tiles))[0] + + if stack is False: + with tempfile.TemporaryDirectory(dir=basepath) as parent_dir: + _ = [shutil.move(fm, parent_dir) for fm in tifs_one_date] + + tifs_all = glob(os.path.join(parent_dir, "*")) + if len(tifs_all) == 0: + raise FileNotFoundError("No files found in {dir}".format(dir=parent_dir)) + + bands = list(set(["_".join(os.path.basename(fa).split("_")[6::]).strip(".tif") for fa in tifs_all])) + for bi in bands: + tifs = glob(os.path.join(parent_dir, "S2*{band}*.tif".format(band=bi))) + if len(tifs) != 0: + __merge_tifs_same_timestamp(tifs, basepath, target_tile=target_tile) + + else: + with tempfile.TemporaryDirectory(dir=basepath) as parent_dir: + _ = [shutil.move(fm, parent_dir) for fm in tifs_one_date] + tifs_all = glob(os.path.join(parent_dir, "*")) + + if len(tifs_all) == 0: + raise FileNotFoundError("No files found in {dir}".format(dir=parent_dir)) + + __merge_tifs_same_timestamp(tifs_all, basepath, target_tile=target_tile) + + else: + raise ValueError("List of tifs to merge to short") def mean_rgb_comb(infiles, rgb_comb=("B04", "B03", "B02"), bad_data_value=np.NaN): @@ -153,7 +287,7 @@ def get_lim(limfn, rgb_comb=("B04", "B03", "B02"), bad_data_value=np.NaN): def mk_rgb(basedir, outdir, rgb_comb=("B04", "B03", "B02"), rgb_gamma=(1.0, 1.0, 1.0), extension="jpg", - resample_order=1, rgb_type=np.uint8, bad_data_value=np.NaN): + resample_order=1, rgb_type=np.uint8, bad_data_value=np.NaN, logger=None): """ Generation of RGB Images from Sentinel-2 data. Use hist_stretch = 'single', if to process a single image. Use hist_stretch = 'series', if to process a sequence of images of the same tile but different dates. @@ -169,10 +303,8 @@ def mk_rgb(basedir, outdir, rgb_comb=("B04", "B03", "B02"), rgb_gamma=(1.0, 1.0, :return: output directory and filename """ - # Option 1 for a series of images fnout_list = [] - - infiles = glob(os.path.join(basedir, "*stack*.tif")) + infiles = glob(os.path.join(basedir, "*.tif")) if len(infiles) == 0: raise FileExistsError("Could not find any stacked tif files in {dir}".format(dir=basedir)) @@ -193,7 +325,6 @@ def mk_rgb(basedir, outdir, rgb_comb=("B04", "B03", "B02"), rgb_gamma=(1.0, 1.0, fn_out = join(outdir, "{pref}_RGB_{bands}.{ext}".format(pref="_".join(split), bands="_".join(rgb_comb), ext=extension)) if isfile(fn_out) is False: - # Create output array ds = gdal.Open(fin, gdal.GA_ReadOnly) sdata = ds.GetRasterBand(1) @@ -202,18 +333,13 @@ def mk_rgb(basedir, outdir, rgb_comb=("B04", "B03", "B02"), rgb_gamma=(1.0, 1.0, S2_rgb = np.zeros(output_shape + [len(rgb_comb), ], dtype=rgb_type) if ds is None: raise ValueError("Can not load {fn}".format(fn=fin)) - # + for i_rgb, (band, gamma) in enumerate(zip(rgb_comb, rgb_gamma)): data = ds.GetRasterBand(i_rgb + 1).ReadAsArray() / 10000.0 # Determine stretching limits - if band == rgb_comb[0]: - lim = limits[0] - if band == rgb_comb[1]: - lim = limits[1] - if band == rgb_comb[2]: - lim = limits[2] + lim = limits[rgb_comb.index(band)] # Calculate and apply zoom factor to input image zoom_factor = np.array(output_shape) / np.array(data[:, :].shape) @@ -224,17 +350,14 @@ def mk_rgb(basedir, outdir, rgb_comb=("B04", "B03", "B02"), rgb_gamma=(1.0, 1.0, # Rescale intensity to range (0.0, 255.0) bf = rescale_intensity(image=zm, in_range=lim, out_range=(0.0, 255.0)) - # Create output image of rgb type and apply gamma correction - S2_rgb[:, :, i_rgb] = np.array(bf, dtype=rgb_type) if gamma != 0.0: - S2_rgb[:, :, i_rgb] = np.array( - adjust_gamma(np.array(S2_rgb[:, :, i_rgb], dtype=np.float32), - gamma), dtype=rgb_type) + S2_rgb[:, :, i_rgb] = np.array(adjust_gamma(bf, gamma), dtype=rgb_type) + else: + S2_rgb[:, :, i_rgb] = np.array(bf, dtype=rgb_type) + del data del ds - # Save output rgb image - makedirs(dirname(fn_out), exist_ok=True) imsave(fn_out, S2_rgb) fnout_list.append(fn_out) @@ -245,19 +368,21 @@ def json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level """ Get data from json dict and save if as singletif files OR: save all requested bands plus cloudmask as one tiff file per date and tile. - :param out_mode: - :param api_result: - :param only_tile: - :param outpath: - :param out_prefix: - :param wl: - :param level: - :param stack_resolution: - :param bands: - :param logger: - :return: + :type out_mode: string + :type api_result: json sring + :type only_tile: string + :type outpath: string + :type out_prefix: string list of + :type wl: dict + :type level: string + :type stack_resolution: string + :type bands: string + :type logger: logger + :return: List of written tifs (list of strings) """ + tif_list = [] + if out_mode == "single": for tile_key in api_result['Results'].keys(): if not ((only_tile != "") & (tile_key != only_tile)): @@ -301,6 +426,7 @@ def json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level img.FlushCache() del img + tif_list.append(outfile) # create tif file for clm if level != "L1C": @@ -322,6 +448,7 @@ def json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level "2016/06 and 2017/12.") img.FlushCache() del img + tif_list.append(clm_outfile) if out_mode == "stack": for tile_key in api_result['Results'].keys(): @@ -377,11 +504,11 @@ def json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level geotrans = (x_min, int(stack_resolution), 0, y_max, 0, -int(stack_resolution)) driver = gdal.GetDriverByName("GTiff") if level == "L1C": - outfile = "{path}/{sensor}_{level}_{pref}_{date}_{tile}_stack_{band}_{res}m.tif".format( + outfile = "{path}/{sensor}_{level}_{pref}_{date}_{tile}_{band}_{res}m.tif".format( path=outpath, pref=out_prefix, date=ac_date, tile=tile_key, band=bands, level=level, res=stack_resolution, sensor=sensor) else: - outfile = "{path}/{sensor}_{level}_{pref}_{date}_{tile}_stack_{band}_MSK_{res}m.tif".format( + outfile = "{path}/{sensor}_{level}_{pref}_{date}_{tile}_{band}_MSK_{res}m.tif".format( path=outpath, pref=out_prefix, date=ac_date, tile=tile_key, band=bands, level=level, res=stack_resolution, sensor=sensor) img = driver.Create(outfile, cols, rows, count_bands + 1, gdal.GDT_Int32) @@ -414,6 +541,10 @@ def json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level img.FlushCache() del img + tif_list.append(outfile) + + return tif_list + def json_to_netcdf(out_mode, api_result, outpath, out_prefix, geo_ll, geo_ur, start_date, end_date, bands, level, wl): """ @@ -514,34 +645,68 @@ def json_to_netcdf(out_mode, api_result, outpath, out_prefix, geo_ll, geo_ur, st raise Exception("Something went wrong while saving as netcdf. " + traceback.format_exc()) +def __get_auth(logger=None): + """ + Gets auth and port + :param logger: logger + :return: dict + """ + + home_dir = expanduser("~") + cred_file = "%s/credentials_gts2_client" % home_dir + if len(glob(cred_file)) == 1: + with open(cred_file, "r") as cf: + cred_dict = json.load(cf) + if "port" in cred_dict.keys(): + port = cred_dict["port"] + else: + port = 80 + auth = (cred_dict["user"], cred_dict["password"]) + else: + logger.error("You did not save your credentials in %s." % cred_file) + user = input("gts2_client - Please insert username: ") + password = input("gts2_client - Please insert password: ") + port = input("gts2_client - Please insert port for API (for default, type 'yes'): ") + if port == "yes": + port = "80" + auth = (user, password) + + return {"auth": auth, "port": port} + + def client(outpath="", out_prefix="", out_mode="json", geo_ll=(), geo_ur=(), sensor="S2A", bands="", max_cloudy="0.5", level="L2A", start_date="", end_date="", version="0.12", suffix="", minimum_fill="", - only_tile="", stack_resolution="", quiet=False, rgb_extension="jpg", rgb_bands_selection="realistic"): + only_tile="", stack_resolution="10", quiet=False, rgb_extension="jpg", rgb_bands_selection="realistic", + merge_tifs=False, merge_tile=None): """ Downloads data via API and saves it in a wanted file format (.json, .tiff or .nc) or alternatively returns a python dictionary. - :param rgb_bands_selection: string - :param rgb_extension: string - :param quiet: boolean - :param only_tile: string - :param minimum_fill: string - :param outpath: string - :param out_prefix: string - :param out_mode: string - :param geo_ll: tuple of floats - :param geo_ur: tuple of floats - :param sensor: string - :param bands: string - :param max_cloudy: string - :param level: string - :param start_date: string - :param end_date: string - :param version: string - :param suffix: string - :param stack_resolution: string + :type merge_tile: string + :type merge_tifs: boolean + :type rgb_bands_selection: string + :type rgb_extension: string + :type quiet: boolean + :type only_tile: string + :type minimum_fill: string + :type outpath: string + :type out_prefix: string + :type out_mode: string + :type geo_ll: tuple of floats + :type geo_ur: tuple of floats + :type sensor: string + :type bands: string + :type max_cloudy: string + :type level: string + :type start_date: string + :type end_date: string + :type version: string + :type suffix: string + :type stack_resolution: string :return: """ + stime = time.time() + wl = {'B01': '443nm', 'B02': '490nm', 'B03': '560nm', 'B04': '665nm', 'B05': '705nm', 'B06': '740nm', 'B07': '783nm', 'B08': '842nm', 'B8A': '865nm', 'B09': '940nm', 'B10': '1375nm', 'B11': '1610nm', 'B12': '2190nm'} @@ -553,53 +718,54 @@ def client(outpath="", out_prefix="", out_mode="json", geo_ll=(), geo_ur=(), sen logging.basicConfig(level=loggerlevel, format='# %(name)-15s %(message)s') logger = logging.getLogger(name="gts2_client") - if stack_resolution not in ["10", "20", ""]: - raise AssertionError("Stack resolution is not in ", ["10", "20"]) + if "_" in out_prefix: + raise ValueError("out_prefix contains '_'. Please use different character, e.g. '-'.") + + valid_stack_resolution = ["10", "20", ""] + if stack_resolution not in valid_stack_resolution: + raise AssertionError("Stack resolution is not in ", valid_stack_resolution) if out_mode == "stack" and "B01" in bands: raise AssertionError("Band 1 (B01) can not be stacked. Please request separately.") + if out_mode != "rgb" and bands == "": + raise AssertionError("Please provide at least one band.") + + valid_out_modes = ["json", "nc", "single", "stack", "python", "rgb"] try: - assert out_mode in ["json", "nc", "single", "stack", "python", "rgb"] + assert out_mode in valid_out_modes except: - raise AssertionError("Invalid output mode. Choose 'json', 'nc', 'single', 'stack' or 'python'.") + raise AssertionError("Invalid output mode. Choose from {list}".format(list=", ".join(valid_out_modes))) - home_dir = expanduser("~") - cred_file = "%s/credentials_gts2_client" % home_dir - if len(glob(cred_file)) == 1: - with open(cred_file, "r") as cf: - cred_dict = json.load(cf) - auth = (cred_dict["user"], cred_dict["password"]) - else: - logger.error("You did not save your credentials in %s." % cred_file) - user = input("gts2_client - Please insert username: ") - password = input("gts2_client - Please insert password: ") - auth = (user, password) + auth = __get_auth(logger=logger) if out_mode == "rgb": band_setting = band_settings[rgb_bands_selection] bands = "_".join(band_setting) + else: + band_setting = [] # options opts = {"ll": geo_ll, "ur": geo_ur, "sensor": sensor, "bands": bands, "max_cloudy": max_cloudy, "auth": auth, "level": level, "date_from": start_date, "date_to": end_date, "version": version, "suffix": suffix, "minimum_fill": minimum_fill, "utm_zone": only_tile, "logger": logger} - # actual request - logger.info(" Requesting data from the GTS2 server ...", ) - stime = time.time() - api_result = gts2_request(**opts) - runtime = time.time() - stime - logger.info(">> Runtime of API %7.2f minutes" % (runtime / 60.)) + # actual API request + logger.info("Requesting data from the GTS2 server ...", ) + a_stime = time.time() + api_result = Gts2Request(opts, logger=logger) + a_runtime = time.time() - a_stime + logger.info(">>>>>> Runtime of API: %7.2f minutes" % (a_runtime / 60.)) - if "message" in api_result.keys(): + if api_result["http_code"] >= 500: logger.error("##################") - logger.error(api_result["message"]) - raise ChildProcessError("API call not right or server Problem.") + raise ChildProcessError("API call not right or server Problem (http_code={code}).".format( + code=api_result["http_code"])) if api_result["ControlValues"]["API_status"] == 1: logger.error("##################") logger.error(str(api_result["ControlValues"]["API_message"])) - raise ChildProcessError("Something went wrong on GTS2 Server.") + raise ChildProcessError("Something went wrong on GTS2 Server (http_code={code}).".format( + code=api_result["http_code"])) if only_tile != "": requ_tiles = api_result['Results'].keys() @@ -619,18 +785,25 @@ def client(outpath="", out_prefix="", out_mode="json", geo_ll=(), geo_ur=(), sen logger.info("Converting data to netcdf-file ...") json_to_netcdf(out_mode, api_result, outpath, out_prefix, geo_ll, geo_ur, start_date, end_date, bands, level, wl) + elif out_mode == "single" or out_mode == "stack": logger.info("Converting data to %s tif-files ..." % out_mode, ) - json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level, stack_resolution, bands, - logger=logger) + tif_list = json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level, stack_resolution, + bands, logger=logger) + if merge_tifs is True: + merge_tiles(tif_list, out_mode=out_mode, target_tile=merge_tile) elif out_mode == "rgb": logger.info("Converting data to %s files ..." % out_mode, ) with tempfile.TemporaryDirectory(dir=outpath) as tmp_dir: - json_to_tiff("stack", api_result, only_tile, tmp_dir, out_prefix, wl, level, stack_resolution, - bands, logger=logger) - mk_rgb(tmp_dir, tmp_dir, rgb_comb=band_setting, extension=rgb_extension) + tif_list = json_to_tiff("stack", api_result, only_tile, tmp_dir, out_prefix, wl, level, stack_resolution, + bands, logger=logger) + + if merge_tifs is True: + merge_tiles(tif_list, out_mode="stack", target_tile=merge_tile) + + mk_rgb(tmp_dir, tmp_dir, rgb_comb=band_setting, extension=rgb_extension, logger=logger) rgb_files = glob(os.path.join(tmp_dir, "*RGB*.{ext}".format(ext=rgb_extension))) _ = [shutil.move(fi, os.path.join(outpath, os.path.basename(fi))) for fi in rgb_files] @@ -638,7 +811,9 @@ def client(outpath="", out_prefix="", out_mode="json", geo_ll=(), geo_ur=(), sen logger.info("Returning python dictionary ...") return api_result - logger.info("....DONE.....") + runtime = time.time() - stime + logger.info("Total runtime: %7.2f minutes" % (runtime / 60.)) + logger.info("....DONE") return @@ -657,7 +832,7 @@ if __name__ == "__main__": parser.add_argument("-o", "--out_dir", action="store", required=True, type=str, help="output_directory") parser.add_argument("-r", "--out_prefix", action="store", required=False, type=str, - help="output_prefix for naming the output files") + help="output_prefix for naming the output files, it should not contain '_'.") parser.add_argument("-m", "--out_mode", action="store", required=False, type=str, default="json", help="output_mode, use 'json' for json-file," "'single' for single tiffs or 'stack' for band stack " @@ -676,7 +851,7 @@ if __name__ == "__main__": help="processing level (e.g. L2A)") parser.add_argument("-v", "--version", action="store", required=False, type=str, default="0.12", help="version of atmospheric correction (e.g. 0.10)") - parser.add_argument("-b", "--bands", action="store", required=False, type=str, + parser.add_argument("-b", "--bands", action="store", required=False, type=str, default="", help="list of Bands (e.g. -b B02_B03") parser.add_argument("-s", "--start_date", action="store", required=True, type=str, help="Startdate e.g. 20160701") @@ -695,8 +870,12 @@ if __name__ == "__main__": parser.add_argument("-n", "--rgb_extension", action="store", required=False, type=str, default="jpg", help="file extension of rgb files e.g.[jpg, png], default: jpg") parser.add_argument("-q", "--rgb_bands_selection", action="store", required=False, type=str, default="realistic", - help="band selection for rgb production, choose from: realistic, nice_looking, vegetation, " - "healthy_vegetation_urban, water, snow, agriculture") + help="band selection for rgb production, choose from: [realistic, nice_looking, vegetation, " + "healthy_vegetation_urban, snow, agriculture]") + parser.add_argument("-w", "--merge_tifs", action="store", required=False, type=str2bool, default=False, + help="Merge tifs and RGBs if area in two or more MGRS tiles per time step (True or False).") + parser.add_argument("-x", "--merge_tile", action="store", required=False, type=str, default=None, + help="Choose MGRS tile into which the merge of files is performed (e.g. 33UUV).") args = parser.parse_args() @@ -719,4 +898,6 @@ if __name__ == "__main__": only_tile=args.utm_zone, stack_resolution=args.stack_resolution, rgb_extension=args.rgb_extension, - rgb_bands_selection=args.rgb_bands_selection) + rgb_bands_selection=args.rgb_bands_selection, + merge_tifs=args.merge_tifs, + merge_tile=args.merge_tile) diff --git a/gts2_client_conda_install.yml b/gts2_client_conda_install.yml new file mode 100644 index 0000000000000000000000000000000000000000..e56f76b2587eff1b6ddcf5f5d8f2717d4ed1d84f --- /dev/null +++ b/gts2_client_conda_install.yml @@ -0,0 +1,15 @@ +name: root +channels: + - conda-forge +dependencies: + - scipy + - requests + - scikit-image + - gdal + - ipython + - pip + - libssh2 + - pip: + - netcdf4 + + diff --git a/install_py_gts2_client.sh b/install_py_gts2_client.sh new file mode 100755 index 0000000000000000000000000000000000000000..41b5ee88e32960c61c81ad9be19bbadf68235497 --- /dev/null +++ b/install_py_gts2_client.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +echo "###################################" +echo "This is the installation routine for installing Miniconda and all necessary packages for gts2_client." +echo "Depending on the load of your machine, this process can take up to an hour, so lean back and relax!" +echo "All files are stored into your installation path which you will provide in the following." +echo "Please make sure that you have sufficient free space in ths installation path (at least 3 GB)." +echo "###################################" +current_path=$(pwd) + +read -p "Please enter path for the python installation: " inst_path +cd ${inst_path} + +conda_vers="miniconda3" +url="https://repo.continuum.io/miniconda/" +conda_inst_name="Miniconda3-latest-Linux-x86_64.sh" +echo "#### Downloading ${conda_vers}.... ####" +wget ${url}${conda_inst_name} +chmod 755 ${conda_inst_name} +echo "#### Installing ${conda_vers}.... ####" +./${conda_inst_name} -p ${inst_path}/${conda_vers} -b +rm -f ${conda_inst_name} +echo "... Ready" + +env_name=gts2_client +echo "Creating a conda environment named: ${env_name} ..." +export PATH=${inst_path}/${conda_vers}/bin/:$PATH +export PYTHONPATH=${inst_path}/${conda_vers}/lib/python3.6/site-packages/ +echo "... Ready" + +echo "Installing necessary packages ..." +conda env update -f ${current_path}/gts2_client_conda_install.yml +echo "... Ready" + +echo "Installing gts2_client ..." +cd ${current_path} +python setup.py install +echo "... Ready" + +echo "###############################" +echo "Congratulations! Miniconda and all necessary packages are installed" +echo " " +echo "###############################" +echo "In order to load your environment and run gts2_client, run the following commands in your bash shell:" +echo "###############################" +echo "### 1. ###" +echo "Set the paths right (you can also add the following lines to your $HOME/.bashrc [recommended]) :" +echo "--------" +echo "export PATH=${inst_path}/${conda_vers}/bin/:$PATH" +echo "export PYTHONPATH=${inst_path}/${conda_vers}/lib/python3.6/site-packages/:$PYTHONPATH" +echo "export LD_LIBRARY_PATH=${inst_path}/${conda_vers}/lib" +echo "--------" +echo "### 2. ###" +echo "Activate your conda environment:" +echo "source activate ${env_name}" +echo " " +echo "... Now you can use the gts2_client ..." +echo " " diff --git a/setup.py b/setup.py index 63ef1c762c855177915c208604a0c34ad52b68d2..0aeb5ab47da1703b880eba095057a9a609738580 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages from importlib import util -requirements = ["numpy", "scipy", "netCDF4", "requests", "scikit-image"] +requirements = ["numpy", "scipy", "netCDF4", "requests", "scikit-image", "gdal"] other_requirements = ["gdal"] test_requirements = requirements + ["coverage"] @@ -17,7 +17,7 @@ if not_installed != []: ', '.join(not_installed))) setup(name='gts2_client', - version='0.4', + version='0.5', packages=find_packages(exclude=['tests*']), url='https://gitext.gfz-potsdam.de/gts2/gts2_client.git', license='GNU LESSER GENERAL PUBLIC LICENSE - Version 3, 29 June 2007', @@ -28,4 +28,4 @@ setup(name='gts2_client', install_requires=requirements, test_suite='tests', tests_require=test_requirements - ) \ No newline at end of file + ) diff --git a/tests/test_gts2_client.py b/tests/test_gts2_client.py index 6dcbfe8d380d21075484d3e854a1e8a1cf3e9974..a60570743fdd05550ff062a0a6d8b876194f68e1 100644 --- a/tests/test_gts2_client.py +++ b/tests/test_gts2_client.py @@ -62,11 +62,44 @@ class TestGts2Client(unittest.TestCase): minimum_fill=minimum_fill, stack_resolution=stack_res) - list_of_files = glob(os.path.join(tmpdir, "*{mode}*.{form}".format(form=out_formats[ii], - mode=out_mode))) + list_of_files = glob(os.path.join(tmpdir, "*.{form}".format(form=out_formats[ii]))) if len(list_of_files) == 0: - raise FileNotFoundError("Could not find files matching: *{mode}*.{form}".format( - form=out_formats[ii], mode=out_mode)) + raise FileNotFoundError("Could not find files matching: *.{form}".format(form=out_formats[ii])) + else: + print("Test OK.") + + def test_gts2_client_merge(self): + """ + Test the merge tifs and rgbs if different tiles are retrieved. + :return: + """ + out_modes = ["single", "stack", "rgb", "stack"] + out_formats = ["tif", "tif", "jpg", "tif"] + levels = ["L2A", "L2A", "L2A", "L1C"] + merge_tile = ["32UQD", "32UQD", None, None] + + for ii, out_mode in enumerate(out_modes): + with tempfile.TemporaryDirectory() as tmpdir: + gts2_client.client(outpath=tmpdir, + out_prefix="krh", + out_mode=out_mode, + geo_ll=(12.559433, 53.036066), + geo_ur=(12.737961, 53.238058), + sensor="S2A", + level=levels[ii], + version="0.12", + bands="B04_B03_B02", + max_cloudy="0.2", + suffix="", + start_date="20170501", + end_date="20170530", + minimum_fill="0.9", + merge_tifs=True, + merge_tile=merge_tile[ii]) + + list_of_files = glob(os.path.join(tmpdir, "*.{form}".format(form=out_formats[ii]))) + if len(list_of_files) == 0: + raise FileNotFoundError("Could not find files matching: *.{form}".format(form=out_formats[ii])) else: print("Test OK.") @@ -162,6 +195,45 @@ class TestGts2Client(unittest.TestCase): :return: """ + try: + out_mode = "stack" + print("#### Testing '_' in prefix ") + gts2_client.client( + out_prefix="test_", + out_mode=out_mode, + geo_ll=geo_ll, + geo_ur=geo_ur, + bands="B05", + start_date=start_date, + end_date=end_date, + version=version, + level=level, + max_cloudy=max_cloudy, + minimum_fill=minimum_fill, + stack_resolution=stack_res, + quiet=False) + except ValueError: + print("Test OK.") + + try: + out_mode = "stack" + print("#### Testing too large area") + gts2_client.client( + out_mode=out_mode, + geo_ll=geo_ll, + geo_ur=(12.737961+0.4, 53.238058+0.4), + bands="B05", + start_date=start_date, + end_date=end_date, + version=version, + level=level, + max_cloudy=max_cloudy, + minimum_fill=minimum_fill, + stack_resolution=stack_res, + quiet=False) + except ValueError: + print("Test OK.") + try: out_mode = "python" print("#### Testing stack_resolution=60") @@ -201,7 +273,7 @@ class TestGts2Client(unittest.TestCase): print("Test OK.") try: - out_mode = "dowsnotwork" + out_mode = "doesnotwork" print("#### Testing wrong out_mode") gts2_client.client( out_mode=out_mode,