diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b7e0c42e515f3919709805e85a02b679150f7bb..29cde43c7b053f51d3e1349b0685bede581bf3e1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,5 +7,19 @@ test_gts2_client: script: - source ~/anaconda3/bin/activate - export GDAL_DATA=/home/gitlab-runner/anaconda3/share/gdal - - python setup.py install - - ./test.sh + - make coverage + artifacts: + paths: + - htmlcov/ + +pages: + stage: deploy + dependencies: + - test_gts2_client + script: + - mkdir -p public/coverage + - cp -r htmlcov/* public/coverage/ + artifacts: + paths: + - public + expire_in: 600 days \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..66d78694c004e4ac1597bd9ad13b9ab06bcaaae9 --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +help: + @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + +clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts + +clean-build: ## remove build artifacts + rm -fr build/ + rm -fr dist/ + rm -fr .eggs/ + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + + +clean-pyc: ## remove Python file artifacts + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + +clean-test: ## remove test and coverage artifacts + rm -rf .tox/ + rm -rf .coverage + rm -rf htmlcov/ + +coverage: ## check code coverage quickly with the default Python + coverage run --source gts2_client setup.py test + coverage report -m + coverage html + +coverage_view: coverage ## check code coverage quickly with the default Python + $(BROWSER) htmlcov/index.htm + +test: ## run tests quickly with the default Python + python setup.py test + +install: clean ## install the package to the active Python's site-packages + python setup.py install + +gitlab_CI_docker: ## Build CI docker runner + cd ./tests/CI_docker/; bash ./build_gts2_client_testsuite_image.sh diff --git a/README.md b/README.md index ec19abf54fa194ff5b9488932793b06b2ee63bf5..2cae5c55dea7150934d5c18d919ddee70f93fb79 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # GTS2 Sentinel-2 downloader -## Status -[![build status](https://gitext.gfz-potsdam.de/gts2/gts2_client/badges/master/build.svg)](https://gitext.gfz-potsdam.de/gts2/gts2_client/commits/master) +Status +------ +.. figure:: https://gitext.gfz-potsdam.de/gts2/gts2_client/master/build.svg +.. figure:: https://gitext.gfz-potsdam.de/gts2/gts2_client/master/coverage.svg ## Description This program package downloads Sentinel-2 data from the GTS2 diff --git a/gts2_client/gts2_client.py b/gts2_client/gts2_client.py index 38160f4a081e2277e662363c031d9ba38ef3f2f0..2da4a22743d1882b155640054b790c16fbfe3fdf 100755 --- a/gts2_client/gts2_client.py +++ b/gts2_client/gts2_client.py @@ -13,13 +13,14 @@ import netCDF4 as nc4 from glob import glob from os.path import expanduser import time +import logging warnings.filterwarnings("ignore") class gts2_request(dict): def __init__(self, ll, ur, auth, bands="B02_B04_B08", version="0.6", date_from="20160107", date_to="20160507", - minimum_fill="1.0", sensor="S2A", level="L2A", suffix="", max_cloudy="1.0", utm_zone=""): + minimum_fill="1.0", sensor="S2A", level="L2A", suffix="", max_cloudy="1.0", utm_zone="", logger=None): if auth[0] == "agricircle": address = "https://rz-vm175.gfz-potsdam.de:4444/AC" @@ -49,15 +50,31 @@ class gts2_request(dict): minimum_fill=minimum_fill, sensor=sensor, level=level, version=version, ll_lon=ll[0], ll_lat=ll[1], ur_lon=ur[0], ur_lat=ur[1], max_cloudy=max_cloudy, utm_zone=utm_zone) - print(">>>>>> API-call: %s " % self.api_call) + logger.info(">>>>>> API-call: %s " % self.api_call) # get data, update dict from json self.update(requests.get(self.api_call, verify=False, auth=auth).json()) -def json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level, stack_resolution, bands): - # get data from json dict and convert to array +def json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level, stack_resolution, bands, logger=None): + """ + Get data from json dict and save if as singletif files OR: + save all requested bands plus cloudmask as one tiff file per date and tile. + :param out_mode: + :param api_result: + :param only_tile: + :param outpath: + :param out_prefix: + :param wl: + :param level: + :param stack_resolution: + :param bands: + :param logger: + :return: + """ + if out_mode == "single": + logger.info("Converting json data to single tiff files") try: for tile_key in api_result['Results'].keys(): if (only_tile != "") & (tile_key != only_tile): @@ -118,10 +135,8 @@ def json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level except: raise Exception("Something went wrong while writing single tifs. " + traceback.format_exc()) - print("... converting json data to single tiff files: ... done") - - # save all requested bands plus cloudmask as one tiff file per date and tile if out_mode == "stack": + logger.info("Converting json data to stacked tiff file") try: for tile_key in api_result['Results'].keys(): # get the number of timesteps: @@ -142,8 +157,8 @@ def json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level stack_resolution in key] if len(find_ref_band_key) == 0: - print("You have chosen %sm for stack resolution, but did not request at least one band with " - "spatial sampling of %sm" % (stack_resolution, stack_resolution)) + logger.error("You have chosen %sm for stack resolution, but did not request at least one " + "band with spatial sampling of %sm" % (stack_resolution, stack_resolution)) break ref_band_key = find_ref_band_key[0] ref_data = api_result['Results'][tile_key][ref_band_key]['data'][ti] @@ -209,11 +224,24 @@ def json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level except: raise Exception("Something went wrong while stacking. " + traceback.format_exc()) - print("... converting json data to stacked tiff file: ... done.") - def json_to_netcdf(out_mode, api_result, outpath, out_prefix, geo_ll, geo_ur, start_date, end_date, bands, level, wl): - # get data from json-dict and save as netcdf file + """ + Get data from json-dict and save as netcdf file + :param out_mode: + :param api_result: + :param outpath: + :param out_prefix: + :param geo_ll: + :param geo_ur: + :param start_date: + :param end_date: + :param bands: + :param level: + :param wl: + :return: + """ + if out_mode == "nc": try: outfile = "%s/%s_ll%s_%s_ur%s_%s_s%s_e%s_b%s.nc" % (outpath, out_prefix, geo_ll[0], geo_ll[1], geo_ur[0], @@ -292,15 +320,14 @@ def json_to_netcdf(out_mode, api_result, outpath, out_prefix, geo_ll, geo_ur, st except: raise Exception("Something went wrong while saving as netcdf. " + traceback.format_exc()) - print("... converting json data to netcdf file: ... done.") - -def client(outpath="", out_prefix="", out_mode="json", geo_ll=(), geo_ur=(), sensor="", bands="", max_cloudy="0.5", - level="", start_date="", end_date="", version="0.10", suffix="", minimum_fill="", - only_tile="", stack_resolution=""): +def client(outpath="", out_prefix="", out_mode="json", geo_ll=(), geo_ur=(), sensor="S2A", bands="", max_cloudy="0.5", + level="L2A", start_date="", end_date="", version="0.12", suffix="", minimum_fill="", + only_tile="", stack_resolution="", quiet=False): """ Downloads data via API and saves it in a wanted file format (.json, .tiff or .nc) or alternatively returns a python dictionary. + :param quiet: boolean :param only_tile: string :param minimum_fill: string :param outpath: string @@ -320,14 +347,30 @@ def client(outpath="", out_prefix="", out_mode="json", geo_ll=(), geo_ur=(), sen :return: """ - if stack_resolution not in ["10", "20"]: - print("Stack resolution is not in ", ["10", "20"]) + wl = {'B01': '443nm', 'B02': '490nm', 'B03': '560nm', 'B04': '665nm', 'B05': '705nm', 'B06': '740nm', + 'B07': '783nm', 'B08': '842nm', 'B8A': '865nm', 'B09': '940nm', 'B10': '1375nm', 'B11': '1610nm', + 'B12': '2190nm'} + + if quiet is True: + loggerlevel = logging.ERROR + else: + loggerlevel = logging.INFO + logging.basicConfig(level=loggerlevel, format='# %(name)-15s %(message)s') + logger = logging.getLogger(name="gts2_client") + + if stack_resolution not in ["10", "20", ""]: + logger.error("Stack resolution is not in ", ["10", "20"]) return if out_mode == "stack" and "B01" in bands: - print("Band 1 (B01) can not be stacked. Please request separately.") + logger.error("Band 1 (B01) can not be stacked. Please request separately.") return + try: + assert out_mode in ["json", "nc", "single", "stack", "python"] + except: + raise AssertionError("Invalid output mode. Choose 'json', 'nc', 'single', 'stack' or 'python'.") + home_dir = expanduser("~") cred_file = "%s/credentials_gts2_client" % home_dir if len(glob(cred_file)) == 1: @@ -335,7 +378,7 @@ def client(outpath="", out_prefix="", out_mode="json", geo_ll=(), geo_ur=(), sen cred_dict = json.load(cf) auth = (cred_dict["user"], cred_dict["password"]) else: - print("You did not save your credentials in %s." % cred_file) + logger.error("You did not save your credentials in %s." % cred_file) user = input("gts2_client - Please insert username: ") password = input("gts2_client - Please insert password: ") auth = (user, password) @@ -343,72 +386,55 @@ def client(outpath="", out_prefix="", out_mode="json", geo_ll=(), geo_ur=(), sen # options opts = {"ll": geo_ll, "ur": geo_ur, "sensor": sensor, "bands": bands, "max_cloudy": max_cloudy, "auth": auth, "level": level, "date_from": start_date, "date_to": end_date, - "version": version, "suffix": suffix, "minimum_fill": minimum_fill, "utm_zone": only_tile} + "version": version, "suffix": suffix, "minimum_fill": minimum_fill, "utm_zone": only_tile, "logger": logger} # actual request - print("### Requesting data from the GTS2 server ...", ) + logger.info(" Requesting data from the GTS2 server ...", ) stime = time.time() api_result = gts2_request(**opts) runtime = time.time() - stime - print("### Runtime of API %7.2f minutes" % (runtime / 60.)) + logger.info(">> Runtime of API %7.2f minutes" % (runtime / 60.)) if "message" in api_result.keys(): - print("###### API call not right or server Problem. ######") - print(api_result["message"]) - print("###################################################") + logger.error("###### API call not right or server Problem. ######") + logger.error(api_result["message"]) return if api_result["ControlValues"]["API_status"] == 1: - print("###### API call not right or server Problem. ######") + logger.error("###### API call not right or server Problem. ######") for mi in api_result["ControlValues"]["API_message"]: - print(mi) - print("###################################################") + logger.error(mi) return + if only_tile != "": + requ_tiles = api_result['Results'].keys() + if only_tile not in requ_tiles: + logger.error("Tile %s not available in results." % only_tile) + return + if out_mode == "json": - print("### Saving data to json file ...", ) + logger.info("Saving data to json file ...", ) json_outfile = "%s/%s_ll%s_%s_ur%s_%s_s%s_e%s_b%s.json" % (outpath, out_prefix, geo_ll[0], geo_ll[1], geo_ur[0], geo_ur[1], start_date, end_date, bands) with open(json_outfile, 'w') as f: json.dump(api_result, f, indent=4) - print("... downloading S2 data from API: ... done.") return elif out_mode == "nc": - print("### Converting data to netcdf-file ...") - elif out_mode == "single" or out_mode == "stack": - print("### Converting data to %s tif-files ..." % out_mode, ) - else: - print("### Returning python dictionary ...") - - # test parameters - try: - assert out_mode in ["json", "nc", "single", "stack", "python"] - except: - raise AssertionError("Invalid output mode. Choose 'json', 'nc', 'single', 'stack' or 'python'.") - - if only_tile != "": - requ_tiles = api_result['Results'].keys() - if only_tile not in requ_tiles: - print("Tile %s not available in results." % only_tile) - return - - wl = {'B01': '443nm', 'B02': '490nm', 'B03': '560nm', 'B04': '665nm', 'B05': '705nm', 'B06': '740nm', - 'B07': '783nm', 'B08': '842nm', 'B8A': '865nm', 'B09': '940nm', 'B10': '1375nm', 'B11': '1610nm', - 'B12': '2190nm'} - - if out_mode == "single" or out_mode == "stack": - json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level, stack_resolution, bands) - - if out_mode == "nc": + logger.info("Converting data to netcdf-file ...") json_to_netcdf(out_mode, api_result, outpath, out_prefix, geo_ll, geo_ur, start_date, end_date, bands, level, wl) - - if out_mode == "python": - print("... Returning python dictionary: ... done") + elif out_mode == "single" or out_mode == "stack": + logger.info("Converting data to %s tif-files ..." % out_mode, ) + json_to_tiff(out_mode, api_result, only_tile, outpath, out_prefix, wl, level, stack_resolution, bands, + logger=logger) + elif out_mode == "python": + logger.info("Returning python dictionary ...") return api_result + else: + return - print("... downloading S2 data from API: ... done") + logger.info("....DONE.....") def str2bool(v): @@ -454,7 +480,7 @@ if __name__ == "__main__": help="get data with corrected pixel shifts (True or False)") parser.add_argument("-z", "--max_cloudy", action="store", required=False, type=str, default="0.5", help="maximal percentage of cloudyness of requested scene (e.g. 0.5)") - parser.add_argument("-f", "--minimum_fill", action="store", required=False, type=str, default="0.10", + parser.add_argument("-f", "--minimum_fill", action="store", required=False, type=str, default="0.12", help="minimal percentage of data in scene (e.g. 1.0)") parser.add_argument("-d", "--utm_zone", action="store", required=False, type=str, default="", help="only return data for specific utm-zone (MGRS-tile, e.g. 33UUV)") diff --git a/setup.py b/setup.py index 943d78ff3117cabcf2a1f603c30eae11860c420b..2c268d1c505b91d6b1e97a4e711cbac95e37535f 100755 --- a/setup.py +++ b/setup.py @@ -1,15 +1,31 @@ -from setuptools import setup +from setuptools import setup, find_packages +from importlib import util -requirements = ["numpy", "gdal", "scipy", "netCDF4"] +requirements = ["numpy", "scipy", "netCDF4"] +other_requirements = ["gdal"] test_requirements = ["coverage"] +# test for packages that do not install well with pip +not_installed = [] +for pi in other_requirements: + is_installed = util.find_spec(pi) + if is_installed is None: + not_installed.append(pi) +if not_installed != []: + raise ModuleNotFoundError( + "Could not find the following packages (please use different installer, e.g. conda): %s" % ( + ', '.join(not_installed))) + setup(name='gts2_client', - version='0.2', - packages=['gts2_client'], - url='git@gitext.gfz-potsdam.de:hannesd/gts2_client.git', + version='0.3', + packages=find_packages(exclude=['tests*']), + url='https://gitext.gfz-potsdam.de/gts2/gts2_client.git', license='GNU LESSER GENERAL PUBLIC LICENSE - Version 3, 29 June 2007', author='Hannes Diedrich, Niklas Bohn, Andre Hollstein', author_email='hannes.diedrich@gfz-potsdam.de', description='Downloads Sentinel-2 data from GTS2 cloud', scripts=['gts2_client/gts2_client.py'], - install_requires=requirements) \ No newline at end of file + install_requires=requirements, + test_suite='tests', + tests_require=test_requirements + ) \ No newline at end of file diff --git a/runner_build_script.sh b/tests/CI_docker/build_gts2_client_testsuite_image.sh similarity index 98% rename from runner_build_script.sh rename to tests/CI_docker/build_gts2_client_testsuite_image.sh index b178ac7d8b910279a0b5a2bd862d924041dd2655..bcd869508c6178df5b7b41016529b783d9b2101e 100755 --- a/runner_build_script.sh +++ b/tests/CI_docker/build_gts2_client_testsuite_image.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -context_dir="./CI_runner" +context_dir="./context" dockerfile="build_runner_image.docker" runner_tag="gts2_client_ci" gitlab_runner="gts2_client_gitlab_CI_runner" diff --git a/CI_runner/build_runner_image.docker b/tests/CI_docker/context/build_runner_image.docker similarity index 63% rename from CI_runner/build_runner_image.docker rename to tests/CI_docker/context/build_runner_image.docker index e4579a56b3acbfef8c9702c6a1f393a3314dce81..a79b30deed2e9bca33ebf090825d80461ff78856 100644 --- a/CI_runner/build_runner_image.docker +++ b/tests/CI_docker/context/build_runner_image.docker @@ -1,13 +1,11 @@ FROM centos:7 -RUN yum update -y -RUN yum install -y wget vim bzip2 +RUN yum update -y && \ + yum install -y wget vim bzip2 gcc gcc-c++ make libgl1-mesa-glx mesa-libGL qt5-qtbase-gui git texlive RUN /bin/bash -i -c "wget https://repo.continuum.io/archive/Anaconda3-4.3.1-Linux-x86_64.sh && \ bash ./Anaconda3-4.3.1-Linux-x86_64.sh -b && \ rm -f /root/Anaconda3-4.3.1-Linux-x86_64.sh" RUN /bin/bash -i -c "source ~/anaconda3/bin/activate && \ - conda install --yes pyqt && \ - conda install --yes -c conda-forge gdal && \ - conda install --yes -c conda-forge 'icu=58.*' lxml && \ + conda install --yes -c conda-forge gdal 'icu=58.*' lxml pyqt coverage && \ pip install netCDF4" RUN mkdir -p /home/tmp/ COPY credentials_gts2_client /root/ \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7c68785e9d0b64e1fe46403c4316a9fe1ea36eeb --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- \ No newline at end of file diff --git a/tests/test_gts2_client.py b/tests/test_gts2_client.py new file mode 100644 index 0000000000000000000000000000000000000000..a22e6b5bceeb9641e4e7fa60433ccdbfa2e76ff4 --- /dev/null +++ b/tests/test_gts2_client.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +from gts2_client import gts2_client +import tempfile +from glob import glob +import os + +geo_ll = (12.559433, 53.036066) +geo_ur = (12.737961, 53.238058) +start_date = "20160420" +end_date = "20160430" +bands = "B02_B05" +version = "0.12" +level = "L2A" +max_cloudy = "0.1" +minimum_fill = "1.0" +stack_res = "10" + + +class TestGts2Client(unittest.TestCase): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_gts2_client_tofiles(self): + + out_modes = ["json", "single", "stack", "nc"] + out_formats = ["json", "tif", "tif", "nc"] + + with tempfile.TemporaryDirectory() as tmpdir: + + for ii, out_mode in enumerate(out_modes): + print("#### Testing {om}".format(om=out_mode)) + gts2_client.client(outpath=tmpdir, + out_prefix=out_mode, + out_mode=out_mode, + geo_ll=geo_ll, + geo_ur=geo_ur, + bands=bands, + start_date=start_date, + end_date=end_date, + version=version, + level=level, + max_cloudy=max_cloudy, + minimum_fill=minimum_fill, + stack_resolution=stack_res) + + list_of_files = glob(os.path.join(tmpdir, "*{mode}*.{form}".format(form=out_formats[ii], + mode=out_mode))) + if len(list_of_files) == 0: + raise FileNotFoundError("Could not find files matching: *{mode}*.{form}".format( + form=out_formats[ii], mode=out_mode)) + + def test_gts2_client_topython(self): + + out_mode = "python" + print("#### Testing {om}".format(om=out_mode)) + res = gts2_client.client(out_mode=out_mode, + geo_ll=geo_ll, + geo_ur=geo_ur, + bands=bands, + start_date=start_date, + end_date=end_date, + version=version, + level=level, + max_cloudy=max_cloudy, + minimum_fill=minimum_fill, + stack_resolution=stack_res) + + necessary_keys = ["Acknowledgement", "ControlValues", "Metadata", "Request", "RequestGeoInfo", "Results"] + + for ki in necessary_keys: + if not (ki in res.keys()): + raise KeyError("{key} is not in returned results.".format(key=ki)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test.sh b/tests/test_gts2_client.sh similarity index 100% rename from test.sh rename to tests/test_gts2_client.sh