From ed89b0045198667726bd763ca0d999daf516926e Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 24 Aug 2022 14:39:30 +0200 Subject: [PATCH] Migrate to pytest. Signed-off-by: Daniel Scheffler --- .coveragerc | 3 ++ .gitlab-ci.yml | 26 ++++++++++------- HISTORY.rst | 6 ++++ Makefile | 28 +++++++++++++------ gms_preprocessing/algorithms/L1C_P.py | 2 +- gms_preprocessing/algorithms/geoprocessing.py | 2 ++ gms_preprocessing/misc/helper_functions.py | 2 +- gms_preprocessing/processing/multiproc.py | 4 +++ setup.py | 2 +- .../context/environment_gms_preprocessing.yml | 10 +++---- tests/test_cli.py | 5 ++++ tests/test_config.py | 5 ++++ tests/test_exception_handler.py | 5 ++++ tests/test_fmask_runner.py | 5 ++++ tests/test_gms_preprocessing.py | 5 ++++ tests/test_input_reader.py | 5 ++++ tests/test_locks.py | 5 ++++ 17 files changed, 93 insertions(+), 27 deletions(-) diff --git a/.coveragerc b/.coveragerc index 8b4f3f5..40105a6 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,8 +3,11 @@ branch = False concurrency = multiprocessing parallel = True +omit = */site-packages/*,*/tests/*,*/.eggs/* [report] +show_missing = True + # Regexes for lines to exclude from consideration exclude_lines = # Have to re-enable the standard pragma diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a61f1c0..466dfad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,19 +36,25 @@ test_gms_preprocessing: - pip check - # run nosetests - - make nosetests + # run tests + - make pytest # create the docs - - pip install -U sphinx_rtd_theme # Read-the-docs theme for SPHINX documentation - - pip install -U sphinx-autodoc-typehints - make docs + artifacts: + expose_as: 'Test and coverage report' paths: - htmlcov/ + - report.html - docs/_build/html/ - - nosetests.html - - nosetests.xml + reports: + coverage_report: + coverage_format: cobertura + path: coverage.xml + junit: report.xml + + expire_in: 30 days when: always @@ -115,7 +121,7 @@ pages: # this job must be called 'pages' to advise GitLab to upload content to - mkdir public - mkdir -p public/doc - mkdir -p public/coverage - - mkdir -p public/nosetests_reports + - mkdir -p public/test_reports # Copy over the docs - cp -r docs/_build/html/* public/doc/ @@ -123,14 +129,14 @@ pages: # this job must be called 'pages' to advise GitLab to upload content to # Copy over the coverage reports - cp -r htmlcov/* public/coverage/ - # Copy over the nosetests reports - - cp nosetests.* public/nosetests_reports/ + # Copy over the test reports + - cp report.html public/test_reports/ # Check if everything is working great - ls -al public - ls -al public/doc - ls -al public/coverage - - ls -al public/nosetests_reports + - ls -al public/test_reports artifacts: paths: - public diff --git a/HISTORY.rst b/HISTORY.rst index 70dddc4..491d306 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,12 @@ History ======= +0.19.6 (coming soon) +-------------------- + +* Migrated test calls from nosetests to pytest and implemented new test report (!13). + + 0.19.5 (2021-11-19) ------------------- diff --git a/Makefile b/Makefile index 0e799ec..f03c3ed 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean clean-test clean-pyc clean-build docs help nosetests +.PHONY: clean clean-test clean-pyc clean-build docs help pytest .DEFAULT_GOAL := help define BROWSER_PYSCRIPT import os, webbrowser, sys @@ -48,8 +48,10 @@ clean-test: ## remove test and coverage artifacts rm -f .coverage rm -fr .coverage.* rm -fr htmlcov/ - rm -fr nosetests.html - rm -fr nosetests.xml + rm -fr report.html + rm -fr report.xml + rm -fr coverage.xml + rm -fr .pytest_cache lint: ## check style with flake8 flake8 --max-line-length=120 gms_preprocessing tests > ./tests/linting/flake8.log || \ @@ -78,12 +80,22 @@ coverage: clean-test ## check code coverage quickly with the default Python coverage html # $(BROWSER) htmlcov/index.html -nosetests: clean-test ## Runs nosetests with coverage, xUnit and nose-html-output +pytest: clean-test ## Runs pytest with coverage and creates coverage and test report ## - puts the coverage results in the folder 'htmlcov' - ## - generates 'nosetests.html' (--with-html) - ## - generates 'nosetests.xml' (--with-xunit) which is currently not visualizable by GitLab - nosetests -vv --with-coverage --cover-package=gms_preprocessing --cover-package=bin --cover-erase --cover-html \ - --cover-html-dir=htmlcov --with-html --with-xunit --rednose --force-color + ## - generates cobertura 'coverage.xml' (needed to show coverage in GitLab MR changes) + ## - generates 'report.html' based on pytest-reporter-html1 + ## - generates JUnit 'report.xml' to show the test report as a new tab in a GitLab MR + ## NOTE: additional options pytest and coverage (plugin pytest-cov) are defined in .pytest.ini and .coveragerc + pytest tests \ + --verbosity=3 \ + --color=yes \ + --tb=short \ + --cov=gms_preprocessing \ + --cov-report html:htmlcov \ + --cov-report term-missing \ + --cov-report xml:coverage.xml \ + --template=html1/index.html --report=report.html \ + --junitxml report.xml docs: ## generate Sphinx HTML documentation, including API docs rm -f docs/gms_preprocessing.rst diff --git a/gms_preprocessing/algorithms/L1C_P.py b/gms_preprocessing/algorithms/L1C_P.py index b6ce1a8..2e192b3 100644 --- a/gms_preprocessing/algorithms/L1C_P.py +++ b/gms_preprocessing/algorithms/L1C_P.py @@ -819,7 +819,7 @@ class AtmCorr(object): max_step=120, # default ecmwf_variables=default_products, processes=0, # singleprocessing - force=False) # dont force download if files already exist + force=False) # don't force download if files already exist t1 = time() self.logger.info("Runtime: %.2f" % (t1 - t0)) for result in results: diff --git a/gms_preprocessing/algorithms/geoprocessing.py b/gms_preprocessing/algorithms/geoprocessing.py index 8d2b33a..905213a 100644 --- a/gms_preprocessing/algorithms/geoprocessing.py +++ b/gms_preprocessing/algorithms/geoprocessing.py @@ -368,6 +368,8 @@ class GEOPROCESSING(object): # import multiprocessing # with multiprocessing.Pool() as pool: # results = pool.map(warp_mp,args) + # pool.close() + # pool.join() # print('warping time', time.time() - t0) # from spectral.io import envi diff --git a/gms_preprocessing/misc/helper_functions.py b/gms_preprocessing/misc/helper_functions.py index 65377c8..29fb005 100644 --- a/gms_preprocessing/misc/helper_functions.py +++ b/gms_preprocessing/misc/helper_functions.py @@ -214,7 +214,7 @@ class mp_SharedNdarray(object): def mp_initializer(globals, globs): """ globs shall be dict with name:value pairs, when executed value will be added to - globals under the name name, if value provides a _init attribute this one is + globals under the same name, if value provides a _init attribute this one is called instead. This makes most sense when called as initializer in a multiprocessing pool, e.g.: diff --git a/gms_preprocessing/processing/multiproc.py b/gms_preprocessing/processing/multiproc.py index 6b5490b..175b8f0 100644 --- a/gms_preprocessing/processing/multiproc.py +++ b/gms_preprocessing/processing/multiproc.py @@ -52,6 +52,8 @@ def MAP(func, args, CPUs=None, flatten_output=False): if CPUs and CPUs > 1 and len(args) > 1: with Pool(CPUs) as pool: results = pool.map(func, args) # always returns a list + pool.close() + pool.join() else: results = [func(argset) for argset in args] # generator does not always work properly here @@ -85,6 +87,8 @@ def imap_unordered(func, args, CPUs=None, flatten_output=False): if CPUs and CPUs > 1 and len(args) > 1: with Pool(CPUs) as pool: results = list(pool.imap_unordered(func, args)) # returns an iterator + pool.close() + pool.join() else: results = [func(argset) for argset in args] # generator does not always work properly here diff --git a/setup.py b/setup.py index 338022f..d5cecce 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ req = [ req_setup = ['setuptools-git'] # needed for package_data version controlled by GIT -req_test = ['coverage', 'nose', 'nose2', 'nose-htmloutput', 'rednose', 'urlchecker'] +req_test = ['pytest', 'pytest-cov', 'pytest-reporter-html1', 'urlchecker'] req_doc = ['sphinx-autodoc-typehint', 'sphinx-argparse', 'sphinx_rtd_theme'] diff --git a/tests/CI_docker/context/environment_gms_preprocessing.yml b/tests/CI_docker/context/environment_gms_preprocessing.yml index 06e2371..87b4b16 100644 --- a/tests/CI_docker/context/environment_gms_preprocessing.yml +++ b/tests/CI_docker/context/environment_gms_preprocessing.yml @@ -51,15 +51,13 @@ dependencies: - tqdm # test and docs requirements - - coverage - flake8 - pydocstyle - pylint - - nose - - nose2 - - nose-htmloutput - - rednose + - pytest + - pytest-cov + - pytest-reporter-html1 - sphinx-autodoc-typehints - sphinx-argparse - sphinx_rtd_theme - - urlchecker + - urlchecker!=0.0.33 # https://github.com/urlstechie/urlchecker-python/issues/84 diff --git a/tests/test_cli.py b/tests/test_cli.py index 30de90e..bd56f2d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -129,3 +129,8 @@ class Test_run_filenames(Base_CLITester.Base_CLITestCase): def setUp(self): super().setUp() self.baseargs = ['filenames', 'LC08_L1TP_193029_20170821_20170911_01_T1.tar.gz'] + + +if __name__ == '__main__': + import pytest + pytest.main() diff --git a/tests/test_config.py b/tests/test_config.py index 0d38bad..6bc0a95 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -129,3 +129,8 @@ class Test_JobConfig(TestCase): # check validity GMSValidator(allow_unknown=True, schema=gms_schema_config_output).validate(params) + + +if __name__ == '__main__': + import pytest + pytest.main() diff --git a/tests/test_exception_handler.py b/tests/test_exception_handler.py index 196cd47..c3e1488 100644 --- a/tests/test_exception_handler.py +++ b/tests/test_exception_handler.py @@ -213,3 +213,8 @@ class Test_ExceptionHandler_Subsystems(BaseTest_ExceptionHandler.Test_ExceptionH # check progress stats: must remain unchanged prog_stats_after = self.get_current_progress_stats() self.assertEqual(prog_stats_before, prog_stats_after) + + +if __name__ == '__main__': + import pytest + pytest.main() diff --git a/tests/test_fmask_runner.py b/tests/test_fmask_runner.py index 06f2cdd..0f62112 100644 --- a/tests/test_fmask_runner.py +++ b/tests/test_fmask_runner.py @@ -122,3 +122,8 @@ class Test_FMASK_Runner_Sentinel2(unittest.TestCase): # os.environ['RIOS_DFLT_DRIVER'] = 'VRT' FMR = FMASK_Runner_Sentinel2(testdata['Sentinel2A_new_style_data'], 'Sentinel-2A') self.assertIsInstance(FMR.calc_cloudMask(), GeoArray) + + +if __name__ == '__main__': + import pytest + pytest.main() diff --git a/tests/test_gms_preprocessing.py b/tests/test_gms_preprocessing.py index 34784e1..7c7d552 100644 --- a/tests/test_gms_preprocessing.py +++ b/tests/test_gms_preprocessing.py @@ -736,3 +736,8 @@ if __name__ == '__main__': # Delete the handlers added to the "log_Test"-logger to ensure that no message is output twice in a row, when # the logger is used again. logger.handlers = [] + + +if __name__ == '__main__': + import pytest + pytest.main() diff --git a/tests/test_input_reader.py b/tests/test_input_reader.py index 577472c..c0856b6 100644 --- a/tests/test_input_reader.py +++ b/tests/test_input_reader.py @@ -93,3 +93,8 @@ class Test_DEM_Creator(unittest.TestCase): except ConnectionRefusedError: warnings.warn("test_index_mediator_query_equals_pgSQL_query() could not been run because " "SpatialIndexMediator refused the connection.") + + +if __name__ == '__main__': + import pytest + pytest.main() diff --git a/tests/test_locks.py b/tests/test_locks.py index 12cb243..5525a83 100644 --- a/tests/test_locks.py +++ b/tests/test_locks.py @@ -101,3 +101,8 @@ class Test_MemoryReserver(unittest.TestCase): def test_with_statement(self): with MemoryReserver(mem2lock_gb=20) as lock: self.assertNotEqual(lock, None) + + +if __name__ == '__main__': + import pytest + pytest.main() -- GitLab