.gitlab-ci.yml 3.67 KB
Newer Older
Daniel Scheffler's avatar
Daniel Scheffler committed
1
before_script:
Daniel Scheffler's avatar
Daniel Scheffler committed
2
  - git lfs pull
Daniel Scheffler's avatar
Daniel Scheffler committed
3

4

5
6
7
8
9
10
11
# Advise GitLab that these environment vars should be loaded from the Variables config.
variables:
    GMS_DB_HOST: SECURE
    GMS_INDEX_HOST: SECURE
    GMS_INDEX_PORT: SECURE


12
13
14
15
16
stages:
    - test
    - deploy


17
test_gms_preprocessing:
18
  stage: test
Daniel Scheffler's avatar
Daniel Scheffler committed
19
  script:
Daniel Scheffler's avatar
Daniel Scheffler committed
20
    - source /root/miniconda3/bin/activate ci_env
21
22
23
    - export GMS_db_host=${GMS_DB_HOST}
    - export GMS_index_host=${GMS_INDEX_HOST}
    - export GMS_index_port=${GMS_INDEX_PORT}
24

25
26
27
28
    - conda install -c conda-forge cartopy  # TODO remove as soon as docker container is rebuilt
    - pip install "geoarray>=0.9.0"  # TODO remove as soon as docker container is rebuilt
    - pip install "arosics>=1.0.0"  # TODO remove as soon as docker container is rebuilt

29
30
31
32
33
34
35
    # update sicor
    # - rm -rf context/sicor
    # - git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor
    # - cd ./context/sicor
    # - make download-tables
    # - python setup.py install
    # - cd ../../
Daniel Scheffler's avatar
Daniel Scheffler committed
36
    # make tests
37
    - pip install -U sicor
38
    # - python -c 'from sicor.tables import get_tables; get_tables(sensor="s2"); get_tables(sensor="l8"); get_tables(sensor="enmap", optional_downloads=("ch4",))'
39

40
    - pip check
41

42
    # run tests
43
    - make nosetests
Daniel Scheffler's avatar
Daniel Scheffler committed
44
45
46
47
48
    - make docs
  artifacts:
    paths:
    - htmlcov/
    - docs/_build/html/
49
50
    - nosetests.html
    - nosetests.xml
51
    when: always
Daniel Scheffler's avatar
Daniel Scheffler committed
52

53

54
55
56
test_gms_preprocessing_install:
  stage: test
  script:
Daniel Scheffler's avatar
Daniel Scheffler committed
57
    - source /root/miniconda3/bin/activate
58
59
    - conda update conda
    - conda create -y -q -c conda-forge --name gms_test python=3
60
    - conda activate gms_test
61
62
63
64
65
66
67
68
69

    # update conda and python
    - conda update -n base -c conda-forge conda
#    - conda update -c conda-forge python

    # avoid package incompatibilities
    # - conda config --set channel_priority strict  # otherwise gdal or libgdal may be installed from defaults channel

    # install some dependencies that cause trouble when installed via pip
70
    - conda install -y -c conda-forge numpy gdal scikit-image pyproj geopandas ipython matplotlib cartopy scikit-learn=0.23.2 shapely pyhdf python-fmask holoviews
71
72
73
74
75
76
77
78
79

    # install not pip-installable deps of arosics
    - conda install -y -c conda-forge pyfftw pykrige

    # install not pip-installable deps of sicor
    - conda install -y -q -c conda-forge glymur pygrib cachetools pyhdf h5py pytables llvmlite numba

    # install gms_preprocessing
    - pip install -e .
80
    - pip check
81
82
83
84
85
86
87
88
89
90
91
92
    - cd ..
    - pwd
    - ls

    # test importability
    - python -c "import gms_preprocessing; print(gms_preprocessing)"
    - python -c "from gms_preprocessing import ProcessController; print(ProcessController)"
  only:
    - master
    - dev


93
94
95
test_styles:
  stage: test
  script:
Daniel Scheffler's avatar
Daniel Scheffler committed
96
    - source /root/miniconda3/bin/activate ci_env
97
98
99
100
101
102
103
104
105
    - make lint
  artifacts:
    paths:
    - tests/linting/flake8.log
    - tests/linting/pycodestyle.log
    - tests/linting/pydocstyle.log
    when: always


106
pages:  # this job must be called 'pages' to advise GitLab to upload content to GitLab Pages
Daniel Scheffler's avatar
Daniel Scheffler committed
107
108
  stage: deploy
  dependencies:
109
    - test_gms_preprocessing
Daniel Scheffler's avatar
Daniel Scheffler committed
110
  script:
111
112
113
114
    # Create the public directory
    - rm -rf public
    - mkdir public
    - mkdir -p public/doc
Daniel Scheffler's avatar
Daniel Scheffler committed
115
    - mkdir -p public/coverage
Daniel Scheffler's avatar
Daniel Scheffler committed
116
    - mkdir -p public/nosetests_reports
117

118
    # Copy over the docs
Daniel Scheffler's avatar
Daniel Scheffler committed
119
    - cp -r docs/_build/html/* public/doc/
120

121
122
    # Copy over the coverage reports
    - cp -r htmlcov/* public/coverage/
123

124
125
    # Copy over the nosetests reports
    - cp nosetests.* public/nosetests_reports/
126

127
128
129
130
131
    # Check if everything is working great
    - ls -al public
    - ls -al public/doc
    - ls -al public/coverage
    - ls -al public/nosetests_reports
Daniel Scheffler's avatar
Daniel Scheffler committed
132
133
134
  artifacts:
    paths:
      - public
135
    expire_in: 10 days
Daniel Scheffler's avatar
Daniel Scheffler committed
136
137
  only:
    - master