diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29cde43c7b053f51d3e1349b0685bede581bf3e1..d6499c4a9a6ff5997acfb4afd9ba48cc1a032e3f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,7 @@ test_gts2_client: script: - source ~/anaconda3/bin/activate - export GDAL_DATA=/home/gitlab-runner/anaconda3/share/gdal + - export PROJ_LIB=/root/anaconda3/share/proj - make coverage artifacts: paths: @@ -22,4 +23,4 @@ pages: artifacts: paths: - public - expire_in: 600 days \ No newline at end of file + expire_in: 600 days diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000000000000000000000000000000000..00619b367ad4eb8311cc9b16b7fac688ffa10596 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +## Release notes +* **2020-09-29:** New scipy, add CHANGELOG.md, fix issues #24, #25, #29 and switched to AC version 0.14. Make this release 1.1.0. +* **2020-07-20:** Added timeout parameter to avoid freezing of client and changed versioning, making this version 1.0.0 +* **2019-05-23:** Switched to AC version 0.13 +* **2018-11-13:** Changed default port to 443 +* **2018-02-20:** Improved output to netCDF file +* **2018-02-16:** Added installation information and installation script for miniconda and all needed packages +* **2018-01-18:** Added docker file and script for building an compatible image and running a container for the client +* **2018-01-17:** Added option for mosaicing/merging tifs and RGBs on client side +* **2018-01-10:** Added output of RGB images into jpg or png, nc-output still in progress diff --git a/README.md b/README.md index d3700ce1a2cf09513f278da301dd8959a606e907..8e739a3ec4b38606a1ec992f141f7457f8903e3e 100755 --- a/README.md +++ b/README.md @@ -8,16 +8,6 @@ Status [Coverage report](http://gts2.gitext.gfz-potsdam.de/gts2_client/coverage/) -## Release notes -* **2020-07-20:** Added timeout parameter to avoid freezing of client and changed versioning, making this version 1.0.0 -* **2019-05-23:** Switched to AC version 0.13 -* **2018-11-13:** Changed default port to 443 -* **2018-02-20:** Improved output to netCDF file -* **2018-02-16:** Added installation information and installation script for miniconda and all needed packages -* **2018-01-18:** Added docker file and script for building an compatible image and running a container for the client -* **2018-01-17:** Added option for mosaicing/merging tifs and RGBs on client side -* **2018-01-10:** Added output of RGB images into jpg or png, nc-output still in progress - ## Description This program package downloads Sentinel-2 data from the GTS2 (GFZ Time Series System for Sentinel-2) cloud for a area of interest, @@ -26,15 +16,8 @@ time of interest and wanted band combination and saves them to geotiff (.tiff) ## Requirements 1. Access to GTS2 API (username, password, port) -1. Python3 -1. Python packages: - * numpy - * gdal - * scipy - * json - * netCDF4 - * requests - * scikit-image +2. Python3 +3. Python packages - check the requirements in [setup.py](./setup.py) ## Download package: Clone the repository with: @@ -100,7 +83,7 @@ The building of the image can take some time (up to 30 Minutes), but once it is ### As command line tool: ```bash -gts2_client.py required_arguments [optional_arguments] +gts2_client required_arguments [optional_arguments] ``` The list of arguments including their default values can be called from the command line with: @@ -136,7 +119,7 @@ The list of arguments including their default values can be called from the comm * -t LEVEL, --level LEVEL processing level (e.g. L2A) * -v VERSION, --version VERSION - version of atmospheric correction (e.g. 0.12) + version of atmospheric correction (e.g. 0.14 - recommended version) * -c COREG, --coreg COREG get coregistrated data with corrected pixel shifts (e.g. True) [pixel shifts between time steps were eliminated by coregistrating diff --git a/docker_gts2_client/build_run_gts2_client_docker.sh b/docker_gts2_client/build_run_gts2_client_docker.sh index 6297808fb24bb22c61b4e75e5267589d06f50be5..6e49a3ee834315acd8bd5859f05651739fb175be 100755 --- a/docker_gts2_client/build_run_gts2_client_docker.sh +++ b/docker_gts2_client/build_run_gts2_client_docker.sh @@ -4,7 +4,8 @@ context_dir="./context" dockerfile="gts2_client.docker" runner_os="centos" runner_iname="gts2_client_runner" -runner_tag="${runner_os}:${runner_iname}" +runner_version="latest" +runner_tag="${runner_os}_${runner_iname}:${runner_version}" container_name="gts2_client" cred_file="$HOME/credentials_gts2_client" out_data_folder="/tmp/gts2_client" @@ -43,4 +44,4 @@ mkdir -p ${out_data_folder} echo "Starting gts2_container, please write files to ${out_data_folder} (setting the -o option of client accordingly)" sudo docker run -it --name ${container_name} -v ${out_data_folder}:${out_data_folder} ${runner_tag} \ bash -i -c "cd /home/gts2_client; git pull origin master; source ~/anaconda3/bin/activate; python setup.py install; -echo "";echo "";echo "";echo "";echo '#######';echo 'This is a shell were you can run the gts2_client :::';echo '#######'; bash" \ No newline at end of file +echo "";echo "";echo "";echo "";echo '#######';echo 'This is a shell were you can run the gts2_client :::';echo '#######'; bash" diff --git a/docker_gts2_client/context/gts2_client.docker b/docker_gts2_client/context/gts2_client.docker index 34bc19a51ad19ff867834fa7ed3b53aca658fb74..ad5f27cb3296178b059d77976bc951bc36378f23 100644 --- a/docker_gts2_client/context/gts2_client.docker +++ b/docker_gts2_client/context/gts2_client.docker @@ -1,14 +1,13 @@ FROM centos:7 RUN yum update -y && \ - yum install -y wget vim bzip2 git -ENV anaconda_dl='Anaconda3-5.0.1-Linux-x86_64.sh' + yum install -y wget vim bzip2 gcc gcc-c++ make libgl1-mesa-glx mesa-libGL qt5-qtbase-gui git texlive +ENV anaconda_dl='Anaconda3-2019.07-Linux-x86_64.sh' RUN /bin/bash -i -c "wget https://repo.continuum.io/archive/$anaconda_dl && \ bash ./$anaconda_dl -b && \ rm -f /root/$anaconda_dl" RUN /bin/bash -i -c "source ~/anaconda3/bin/activate && \ - conda install --yes -q -c conda-forge gdal 'icu=58.*' lxml pyqt && \ - pip install netCDF4 && \ - conda update -q --all" + conda config --add channels conda-forge && \ + conda install --yes -c conda-forge proj gdal=3.1.0 lxml pyqt netcdf4 coverage libiconv imageio" RUN /bin/bash -i -c "cd /home/ && \ git clone https://gitext.gfz-potsdam.de/gts2/gts2_client.git" -COPY credentials_gts2_client /root/credentials_gts2_client \ No newline at end of file +COPY credentials_gts2_client /root/credentials_gts2_client diff --git a/gts2_client/gts2_client.py b/gts2_client/gts2_client.py index 0abbbfac2603cdb8b16920495e43ef88417a24f3..b5436af7d9947d0a99136e9e0f7aceeda52ce4aa 100755 --- a/gts2_client/gts2_client.py +++ b/gts2_client/gts2_client.py @@ -23,7 +23,7 @@ from os.path import join from os.path import expanduser from scipy.ndimage.interpolation import zoom from skimage.exposure import rescale_intensity, adjust_gamma -from scipy.misc import imsave +from imageio import imwrite from subprocess import Popen, PIPE band_settings = {"realistic": ("B04", "B03", "B02"), @@ -361,7 +361,7 @@ def mk_rgb(basedir, outdir, rgb_comb=("B04", "B03", "B02"), rgb_gamma=(1.0, 1.0, del data del ds - imsave(fn_out, S2_rgb) + imwrite(fn_out, S2_rgb) fnout_list.append(fn_out) return fnout_list @@ -615,9 +615,9 @@ def json_to_netcdf(out_mode, api_result, outpath, out_prefix, geo_ll, geo_ur, st for attr in api_result[key][tile][band][data_info].keys()] if data_info == 'data': band_arr = np.asarray(api_result[key][tile][band][data_info]) - band_group.createDimension('x', band_arr.shape[2]) + band_group.createDimension('x', band_arr.shape[0]) band_group.createDimension('y', band_arr.shape[1]) - band_group.createDimension('t', band_arr.shape[0]) + band_group.createDimension('t', band_arr.shape[2]) fill = api_result[key][tile][band]['fill_value'] data = band_group.createVariable('Data', 'i4', ('x', 'y', 't'), fill_value=fill) data.units = "None" @@ -666,9 +666,9 @@ def json_to_netcdf(out_mode, api_result, outpath, out_prefix, geo_ll, geo_ur, st api_result[key][tile][msk][data_info]['time']) if data_info == 'data': mask_arr = np.asarray(api_result[key][tile][msk][data_info]) - mask_group.createDimension('x', mask_arr.shape[2]) + mask_group.createDimension('x', mask_arr.shape[0]) mask_group.createDimension('y', mask_arr.shape[1]) - mask_group.createDimension('t', mask_arr.shape[0]) + mask_group.createDimension('t', mask_arr.shape[2]) fill = api_result[key][tile][msk]['fill_value'] data = mask_group.createVariable('Data', 'i4', ('x', 'y', 't'), fill_value=fill) data.units = "None" diff --git a/gts2_client_conda_install.yml b/gts2_client_conda_install.yml index e56f76b2587eff1b6ddcf5f5d8f2717d4ed1d84f..a4e302962f8313acea309e7583d9cad80d5627f1 100644 --- a/gts2_client_conda_install.yml +++ b/gts2_client_conda_install.yml @@ -5,11 +5,12 @@ dependencies: - scipy - requests - scikit-image - - gdal + - proj + - gdal=3.1.0 - ipython - pip - libssh2 - - pip: - - netcdf4 + - netcdf4 + - imageio diff --git a/setup.py b/setup.py index cf04490436203cacb711b06f2cde556d0a8f6a03..c40b5c2ab52721016f9fd503a63666260af8e871 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages from importlib import util -requirements = ["numpy", "scipy", "netCDF4", "requests", "scikit-image", "gdal"] +requirements = ["proj", "numpy", "imageio", "scipy", "netCDF4", "requests", "scikit-image", "gdal"] other_requirements = ["gdal"] test_requirements = requirements + ["coverage"] @@ -17,14 +17,18 @@ if not_installed != []: ', '.join(not_installed))) setup(name='gts2_client', - version='1.0.0', + version='1.1.0', packages=find_packages(exclude=['tests*']), url='https://gitext.gfz-potsdam.de/gts2/gts2_client.git', license='GNU LESSER GENERAL PUBLIC LICENSE - Version 3, 29 June 2007', author='Hannes Diedrich, Niklas Bohn, Andre Hollstein', author_email='hannes.diedrich@gfz-potsdam.de', description='Downloads Sentinel-2 data from GTS2 cloud', - scripts=['gts2_client/gts2_client.py'], + entry_points={ + 'console_scripts': [ + 'gts2_client = gts2_client.gts2_client:main' + ], + }, install_requires=requirements, test_suite='tests', tests_require=test_requirements diff --git a/tests/CI_docker/build_gts2_client_testsuite_image.sh b/tests/CI_docker/build_gts2_client_testsuite_image.sh index bcd869508c6178df5b7b41016529b783d9b2101e..2b3c9811c9e1106f52479c13177406aa3cf39fd5 100755 --- a/tests/CI_docker/build_gts2_client_testsuite_image.sh +++ b/tests/CI_docker/build_gts2_client_testsuite_image.sh @@ -2,7 +2,8 @@ context_dir="./context" dockerfile="build_runner_image.docker" -runner_tag="gts2_client_ci" +runner_version="latest" +runner_tag="gts2_client_ci:$runner_version" gitlab_runner="gts2_client_gitlab_CI_runner" echo "Preparing gts2 credentials file" @@ -37,5 +38,6 @@ sudo docker exec -it ${gitlab_runner} /bin/bash -c "export RUNNER_EXECUTOR=docke --run-untagged=true \ --locked=true \ --tag-list gts2_client \ + --docker-pull-policy='never' \ --description '${runner_name}' \ - --docker-image '${runner_tag}:latest' " + --docker-image '${runner_tag}' " diff --git a/tests/CI_docker/context/build_runner_image.docker b/tests/CI_docker/context/build_runner_image.docker index a79b30deed2e9bca33ebf090825d80461ff78856..08e3afe276528d1b34eb2ca89033de6c20791029 100644 --- a/tests/CI_docker/context/build_runner_image.docker +++ b/tests/CI_docker/context/build_runner_image.docker @@ -1,11 +1,12 @@ FROM centos:7 RUN yum update -y && \ yum install -y wget vim bzip2 gcc gcc-c++ make libgl1-mesa-glx mesa-libGL qt5-qtbase-gui git texlive -RUN /bin/bash -i -c "wget https://repo.continuum.io/archive/Anaconda3-4.3.1-Linux-x86_64.sh && \ - bash ./Anaconda3-4.3.1-Linux-x86_64.sh -b && \ - rm -f /root/Anaconda3-4.3.1-Linux-x86_64.sh" +ENV anaconda_dl='Anaconda3-2019.07-Linux-x86_64.sh' +RUN /bin/bash -i -c "wget https://repo.continuum.io/archive/$anaconda_dl && \ + bash ./$anaconda_dl -b && \ + rm -f /root/$anaconda_dl" RUN /bin/bash -i -c "source ~/anaconda3/bin/activate && \ - conda install --yes -c conda-forge gdal 'icu=58.*' lxml pyqt coverage && \ - pip install netCDF4" + conda config --add channels conda-forge && \ + conda install --yes -c conda-forge proj gdal=3.1.0 lxml pyqt netcdf4 coverage libiconv imageio" RUN mkdir -p /home/tmp/ -COPY credentials_gts2_client /root/ \ No newline at end of file +COPY credentials_gts2_client /root/