From deb816a872acb9734b66b71c93d4e606cfefe4a8 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 15 Dec 2021 23:03:32 +0100 Subject: [PATCH 1/5] Migrate unittest test calls from nosetests to pytest. Signed-off-by: Daniel Scheffler --- .coveragerc | 3 +++ .gitlab-ci.yml | 21 ++++++++++----- Makefile | 26 ++++++++++++++++--- setup.py | 2 +- .../CI_docker/context/environment_arosics.yml | 8 +++--- 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/.coveragerc b/.coveragerc index 8b4f3f5..40105a6 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,8 +3,11 @@ branch = False concurrency = multiprocessing parallel = True +omit = */site-packages/*,*/tests/*,*/.eggs/* [report] +show_missing = True + # Regexes for lines to exclude from consideration exclude_lines = # Have to re-enable the standard pragma diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 041fb72..bd52f77 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,21 +13,28 @@ test_arosics: script: - source /root/mambaforge/bin/activate ci_env + - pip install pytest pytest-cov pytest-reporter-html1 # TODO remove after recreating CI runner + # update py_tools_ds and geoarray - pip install -U py_tools_ds -q - pip install -U geoarray -q # run tests - - make nosetests + - make pytest # create the docs - make docs artifacts: + expose_as: 'Test and coverage report' paths: - htmlcov/ + - report.html - docs/_build/html/ - - nosetests.html - - nosetests.xml + reports: + cobertura: coverage.xml + junit: report.xml + + expire_in: 30 days when: always @@ -87,7 +94,7 @@ pages: # this job must be called 'pages' to advise GitLab to upload content to - mkdir -p public/doc - mkdir -p public/images/ - mkdir -p public/coverage - - mkdir -p public/nosetests_reports + - mkdir -p public/test_reports # Copy over the docs - cp -r docs/_build/html/* public/doc/ @@ -96,14 +103,14 @@ pages: # this job must be called 'pages' to advise GitLab to upload content to # Copy over the coverage reports - cp -r htmlcov/* public/coverage/ - # Copy over the nosetests reports - - cp nosetests.* public/nosetests_reports/ + # Copy over the test reports + - cp report.html public/test_reports/ # Check if everything is working great - ls -al public - ls -al public/doc - ls -al public/coverage - - ls -al public/nosetests_reports + - ls -al public/test_reports artifacts: paths: - public diff --git a/Makefile b/Makefile index e8ca308..43209a4 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean clean-test clean-pyc clean-build docs help nosetests +.PHONY: clean clean-test clean-pyc clean-build docs help pytest .DEFAULT_GOAL := help define BROWSER_PYSCRIPT import os, webbrowser, sys @@ -49,8 +49,11 @@ clean-test: ## remove test and coverage artifacts rm -f .coverage rm -fr .coverage.* rm -fr htmlcov/ - rm -fr nosetests.html - rm -fr nosetests.xml + rm -fr report.html + rm -fr report.xml + rm -fr coverage.xml + rm -fr .pytest_cache + lint: ## check style with flake8 flake8 --max-line-length=120 . tests > ./tests/linting/flake8.log || \ @@ -84,6 +87,23 @@ nosetests: clean-test ## Runs nosetests with coverage, xUnit and nose-html-outpu nosetests -vv --with-coverage --cover-package=arosics --cover-erase --cover-html --cover-html-dir=htmlcov \ --with-html --with-xunit --rednose --force-color +pytest: clean-test ## Runs pytest with coverage and creates coverage and test report + ## - puts the coverage results in the folder 'htmlcov' + ## - generates cobertura 'coverage.xml' (needed to show coverage in GitLab MR changes) + ## - generates 'report.html' based on pytest-reporter-html1 + ## - generates JUnit 'report.xml' to show the test report as a new tab in a GitLab MR + ## NOTE: additional options pytest and coverage (plugin pytest-cov) are defined in .pytest.ini and .coveragerc + pytest tests \ + --verbosity=3 \ + --color=yes \ + --tb=short \ + --cov=arosics \ + --cov-report html:htmlcov \ + --cov-report term-missing \ + --cov-report xml:coverage.xml \ + --template=html1/index.html --report=report.html \ + --junitxml report.xml + docs: ## generate Sphinx HTML documentation, including API docs rm -f docs/arosics.rst rm -f docs/modules.rst diff --git a/setup.py b/setup.py index 8e1e421..208bee1 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ req_setup = [ req_intplot = ['holoviews', 'ipython'] -req_test = ['coverage', 'nose', 'nose2', 'nose-htmloutput', 'rednose', 'urlchecker'] + req_intplot +req_test = ['pytest', 'pytest-cov', 'pytest-reporter-hmtl1', 'urlchecker'] + req_intplot req_doc = ['sphinx-argparse', 'sphinx_rtd_theme', 'sphinx-autodoc-typehints'] diff --git a/tests/CI_docker/context/environment_arosics.yml b/tests/CI_docker/context/environment_arosics.yml index 303bc3c..a47b60e 100644 --- a/tests/CI_docker/context/environment_arosics.yml +++ b/tests/CI_docker/context/environment_arosics.yml @@ -31,15 +31,13 @@ dependencies: - geojson # doc requirements - - coverage - flake8 - - nose - - nose2 - - nose-htmloutput - pycodestyle - pydocstyle - pylint - - rednose + - pytest + - pytest-cov + - pytest-reporter-html1 - sphinx-argparse - sphinx-autodoc-typehints - sphinx_rtd_theme -- GitLab From c372d50ac4254b49b0ed9434b37d520cc1a8720f Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 15 Dec 2021 23:04:25 +0100 Subject: [PATCH 2/5] Properly close and join multiprocessing.Pool to make coverage work in multiprocessing. Signed-off-by: Daniel Scheffler --- arosics/Tie_Point_Grid.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arosics/Tie_Point_Grid.py b/arosics/Tie_Point_Grid.py index ebae29b..4d05653 100755 --- a/arosics/Tie_Point_Grid.py +++ b/arosics/Tie_Point_Grid.py @@ -403,6 +403,8 @@ class Tie_Point_Grid(object): # COREG and is not raised results = results.get() break + pool.close() # needed to make coverage work in multiprocessing + pool.join() else: # declare global variables needed for self._get_spatial_shifts() @@ -974,6 +976,8 @@ class Tie_Point_Grid(object): # # with multiprocessing.Pool() as pool: # self.kriged = pool.map(self.Kriging_mp,args_kwargs_dicts) + # pool.close() # needed to make coverage work in multiprocessing + # pool.join() # else: # self.Kriging_sp(attrName,skip_nodata=skip_nodata,skip_nodata_col=skip_nodata_col, # outGridRes=outGridRes,fName_out=fName_out,tilepos=tilepos) -- GitLab From d16d7ba9692b64d98ff24d90f404214407603fba Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 15 Dec 2021 23:06:13 +0100 Subject: [PATCH 3/5] Updated HISTORY.rst. Signed-off-by: Daniel Scheffler --- HISTORY.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 68e53ff..8098da2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,12 @@ History ======= +1.7.4 (2021-12-15) +------------------ + +* Migrated test calls from nosetests to pytest and implemented new test report (!24). + + 1.7.3 (2021-12-02) ------------------ -- GitLab From c27aa8754865e2ebd2054a5bd92a81d31453dc19 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 15 Dec 2021 23:09:08 +0100 Subject: [PATCH 4/5] Replace nose2.main() with pytest.main(). Signed-off-by: Daniel Scheffler --- tests/test_COREG.py | 4 ++-- tests/test_COREG_LOCAL.py | 4 ++-- tests/test_tie_point_grid.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_COREG.py b/tests/test_COREG.py index c47a546..9f6e356 100644 --- a/tests/test_COREG.py +++ b/tests/test_COREG.py @@ -359,5 +359,5 @@ class CompleteWorkflow_INTER1_S2A_S2A(unittest.TestCase): if __name__ == '__main__': - import nose2 - nose2.main() + import pytest + pytest.main() diff --git a/tests/test_COREG_LOCAL.py b/tests/test_COREG_LOCAL.py index 2508c90..8c78141 100644 --- a/tests/test_COREG_LOCAL.py +++ b/tests/test_COREG_LOCAL.py @@ -201,5 +201,5 @@ class CompleteWorkflow_INTER1_S2A_S2A(unittest.TestCase): if __name__ == '__main__': - import nose2 - nose2.main() + import pytest + pytest.main() diff --git a/tests/test_tie_point_grid.py b/tests/test_tie_point_grid.py index d435bf4..3773ee7 100644 --- a/tests/test_tie_point_grid.py +++ b/tests/test_tie_point_grid.py @@ -134,5 +134,5 @@ class Test_Tie_Point_Grid(unittest.TestCase): if __name__ == '__main__': - import nose2 - nose2.main() + import pytest + pytest.main() -- GitLab From 7536a8fea1569738ecd49b8caaae241953930194 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 15 Dec 2021 23:15:34 +0100 Subject: [PATCH 5/5] Fixed dead link. Signed-off-by: Daniel Scheffler --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 8581d95..7645479 100644 --- a/README.rst +++ b/README.rst @@ -100,7 +100,7 @@ Credits ------- AROSICS was developed by Daniel Scheffler (German Research Centre of Geosciences) within the context of the -`GeoMultiSens `__ project funded by the German Federal Ministry of Education and Research +`GeoMultiSens `__ project funded by the German Federal Ministry of Education and Research (project grant code: 01 IS 14 010 A-C). This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. -- GitLab