From b58b4e2080ed02289c0d96bfc2958de239d192b0 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Thu, 27 Apr 2023 14:42:56 +0100
Subject: [PATCH] work to make full integration tests share more code
---
tests/integration/full/full_test_env_suit.py | 4 +-
tests/integration/full/full_test_epi.py | 65 ++++-----------
tests/integration/full/full_test_survey.py | 83 ++++++++++---------
.../partial/integration_test_utils.py | 4 +-
.../test_suites/advisory_test_suite.py | 2 +-
.../integration/test_suites/epi_test_suite.py | 39 +++++----
.../test_suites/survey_test_suite.py | 9 ++
7 files changed, 96 insertions(+), 110 deletions(-)
diff --git a/tests/integration/full/full_test_env_suit.py b/tests/integration/full/full_test_env_suit.py
index 5505e05..e94a4e8 100644
--- a/tests/integration/full/full_test_env_suit.py
+++ b/tests/integration/full/full_test_env_suit.py
@@ -45,14 +45,12 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
@staticmethod
def run_env_pipeline():
- component = 'Environment'
- IntegrationTestUtils.run_external_pipeline(component,
+ IntegrationTestUtils.run_external_pipeline(BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_COMPONENT_NAME,
IntegrationTestUtils.TEST_START_DATE,
IntegrationTestUtils.EMAIL_CRED_PATH)
if __name__ == '__main__':
-
IntegrationTestUtils.run_fill_integration_test_pipeline(FullTestEnvSuit,
test_prefix = "env",
processor_dir = BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_PROCESSOR_DIR)
diff --git a/tests/integration/full/full_test_epi.py b/tests/integration/full/full_test_epi.py
index ae7d51c..f6077a9 100644
--- a/tests/integration/full/full_test_epi.py
+++ b/tests/integration/full/full_test_epi.py
@@ -22,13 +22,13 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
self.set_expected_values()
- if BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH is None or not os.path.isdir(BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH):
+ if IntegrationTestUtils.TEST_OUT_PATH is None or not os.path.isdir(IntegrationTestUtils.TEST_OUT_PATH):
# if True:
FullTestEpi.write_temp_run_config_files()
FullTestEpi.run_dependent_pipelines()
FullTestEpi.run_epi_pipeline()
else:
- print(f"output in {BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH} already written, skipping rerun")
+ print(f"output in {IntegrationTestUtils.TEST_OUT_PATH} already written, skipping rerun")
@staticmethod
@@ -36,12 +36,12 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
run_dict: dict = copy.deepcopy(default_config_dict)
- run_dict['WorkspacePathout'] = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH
- run_dict['WorkspacePath'] = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH
+ run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
+ run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
# we need to run the env suit pipeline as well
- run_dict['Environment']['WORK_PATH'] = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH
- run_dict['Environment']['INPUT_PATH'] = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH
- run_dict['Environment']['OUTPUT_PATH'] = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH
+ run_dict['Environment']['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
+ run_dict['Environment']['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
+ run_dict['Environment']['OUTPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
#make sure we activate server download
run_dict['Survey']['SkipServerDownload'] = False
@@ -51,25 +51,25 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
run_dict['Epidemiology']['Deposition']['FileListerFunction'] = "list_onefile_operational"
run_dict['Epidemiology']['Environment']['FileListerFunction'] = "list_onefile_operational"
# run_dict['Epidemiology']['Epi']['rescale_output_by_host_raster'] = False
- previous_day_string: str = IntegrationTestUtils.get_day_before_as_string(BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE)
+ previous_day_string: str = IntegrationTestUtils.get_day_before_as_string(IntegrationTestUtils.TEST_START_DATE)
run_dict['Survey']['SeasonStartString'] = previous_day_string
# may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
- os.makedirs(BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH + "temp_config.json"
+ os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
+ IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
@staticmethod
def run_dependent_pipelines():
IntegrationTestUtils.run_external_pipeline("Deposition",
- BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE,
+ IntegrationTestUtils.TEST_START_DATE,
IntegrationTestUtils.EMAIL_CRED_PATH)
IntegrationTestUtils.run_external_pipeline("Environment",
- BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE,
+ IntegrationTestUtils.TEST_START_DATE,
IntegrationTestUtils.EMAIL_CRED_PATH)
- # previous_day_string: str = IntegrationTestUtils.get_day_before_as_string(BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE)
+ # previous_day_string: str = IntegrationTestUtils.get_day_before_as_string(IntegrationTestUtils.TEST_START_DATE)
# IntegrationTestUtils.run_external_pipeline("Survey",
# previous_day_string,
# IntegrationTestUtils.EMAIL_CRED_PATH)
@@ -77,42 +77,13 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
@staticmethod
def run_epi_pipeline():
- component = 'Epidemiology'
- IntegrationTestUtils.run_external_pipeline(component,
- BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE,
+ IntegrationTestUtils.run_external_pipeline(BaseEpiTestSuite.EpiTestSuite.EPI_COMPONENT_NAME,
+ IntegrationTestUtils.TEST_START_DATE,
IntegrationTestUtils.EMAIL_CRED_PATH)
if __name__ == '__main__':
- _parser = IntegrationTestUtils.build_arg_parser()
-
- _args = _parser.parse_args()
- _config_file: str = _args.config
- _outdir: str = _args.outdir
- _email_cred_path: str = _args.email_cred
- _test_report_dir: str = _args.test_report_dir
- _run_date_type: str = _args.run_date_type
- _custom_run_date: str = _args.custom_run_date
-
- IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH = _config_file
-
- nowstring: str = IntegrationTestUtils.get_now_string()
- # prefix: str = "temp_epi_" + nowstring
- prefix: str = "temp_epi"
- BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH = _outdir + prefix + os.sep
- IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path
-
- BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE = IntegrationTestUtils.generate_run_date(_run_date_type, _custom_run_date)
-
- BaseEpiTestSuite.EpiTestSuite.TEST_JOB_DIR = "EPI_" + BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE
-
- # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
- sys.argv[1:] = _args.unittest_args
- tests: TestSuite = TestLoader().loadTestsFromTestCase(FullTestEpi)
-
- if _test_report_dir is None:
- _test_report_dir = BaseEpiTestSuite.EpiTestSuite.TEST_JOB_DIR
-
- runner = HTMLTestRunner(output=_test_report_dir, combine_reports = True)
- runner.run(tests)
+ IntegrationTestUtils.run_fill_integration_test_pipeline(FullTestEpi,
+ test_prefix = "epi",
+ processor_dir = BaseEpiTestSuite.EpiTestSuite.EPI_PROCESSOR_DIR)
diff --git a/tests/integration/full/full_test_survey.py b/tests/integration/full/full_test_survey.py
index 9aafa78..088fbfc 100644
--- a/tests/integration/full/full_test_survey.py
+++ b/tests/integration/full/full_test_survey.py
@@ -22,12 +22,12 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
self.set_expected_values()
- if BaseSurveyTestSuite.SurveyTestSuite.TEST_OUT_PATH is None or not os.path.isdir(
- BaseSurveyTestSuite.SurveyTestSuite.TEST_OUT_PATH):
+ if IntegrationTestUtils.TEST_OUT_PATH is None or not os.path.isdir(
+ IntegrationTestUtils.TEST_OUT_PATH):
FullTestSurvey.write_temp_run_config_file()
FullTestSurvey.run_survey_pipeline()
else:
- print(f"output in {BaseSurveyTestSuite.SurveyTestSuite.TEST_OUT_PATH} already written, skipping rerun")
+ print(f"output in {IntegrationTestUtils.TEST_OUT_PATH} already written, skipping rerun")
@staticmethod
@@ -35,52 +35,55 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
run_dict: dict = copy.deepcopy(default_config_dict)
- run_dict['WorkspacePathout'] = FullTestSurvey.TEST_OUT_PATH
- run_dict['WorkspacePath'] = FullTestSurvey.TEST_OUT_PATH
+ run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
+ run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
run_dict['Survey']['SkipServerDownload'] = False
- os.makedirs(BaseSurveyTestSuite.SurveyTestSuite.TEST_OUT_PATH)
- IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = BaseSurveyTestSuite.SurveyTestSuite.TEST_OUT_PATH + "temp_config.json"
+ os.makedirs(IntegrationTestUtils.TEST_OUT_PATH)
+ IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
@staticmethod
def run_survey_pipeline():
- component = 'Survey'
- IntegrationTestUtils.run_external_pipeline(component,
- FullTestSurvey.TEST_START_DATE,
+ IntegrationTestUtils.run_external_pipeline(BaseSurveyTestSuite.SurveyTestSuite.SURVEY_COMPONENT_NAME,
+ IntegrationTestUtils.TEST_START_DATE,
IntegrationTestUtils.EMAIL_CRED_PATH)
if __name__ == '__main__':
- _parser = IntegrationTestUtils.build_arg_parser()
+ # _parser = IntegrationTestUtils.build_arg_parser()
+ #
+ # _args = _parser.parse_args()
+ # _config_file: str = _args.config
+ # _outdir: str = _args.outdir
+ # _email_cred_path: str = _args.email_cred
+ # _test_report_dir: str = _args.test_report_dir
+ # _run_date_type: str = _args.run_date_type
+ # _custom_run_date: str = _args.custom_run_date
+ #
+ # IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH = _config_file
+ #
+ # nowstring: str = IntegrationTestUtils.get_now_string()
+ # prefix: str = "temp_survey_" + nowstring
+ # # prefix: str = "temp_survey"
+ # BaseSurveyTestSuite.SurveyTestSuite.TEST_OUT_PATH = _outdir + prefix + os.sep
+ # IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path
+ #
+ # BaseSurveyTestSuite.SurveyTestSuite.TEST_START_DATE = IntegrationTestUtils.generate_run_date(_run_date_type, _custom_run_date)
+ #
+ # BaseSurveyTestSuite.SurveyTestSuite.TEST_JOB_DIR = "SURVEY_" + BaseSurveyTestSuite.SurveyTestSuite.TEST_START_DATE
+ #
+ # # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
+ # sys.argv[1:] = _args.unittest_args
+ # tests: TestSuite = TestLoader().loadTestsFromTestCase(FullTestSurvey)
+ #
+ # if _test_report_dir is None:
+ # _test_report_dir = BaseSurveyTestSuite.SurveyTestSuite.TEST_JOB_DIR
+ #
+ # runner = HTMLTestRunner(output=_test_report_dir, combine_reports = True)
+ # runner.run(tests)
+ IntegrationTestUtils.run_fill_integration_test_pipeline(FullTestSurvey,
+ test_prefix = "survey",
+ processor_dir = BaseSurveyTestSuite.SurveyTestSuite.SURVEY_PROCESSOR_DIR)
- _args = _parser.parse_args()
- _config_file: str = _args.config
- _outdir: str = _args.outdir
- _email_cred_path: str = _args.email_cred
- _test_report_dir: str = _args.test_report_dir
- _run_date_type: str = _args.run_date_type
- _custom_run_date: str = _args.custom_run_date
-
- IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH = _config_file
-
- nowstring: str = IntegrationTestUtils.get_now_string()
- prefix: str = "temp_survey_" + nowstring
- # prefix: str = "temp_survey"
- BaseSurveyTestSuite.SurveyTestSuite.TEST_OUT_PATH = _outdir + prefix + os.sep
- IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path
-
- BaseSurveyTestSuite.SurveyTestSuite.TEST_START_DATE = IntegrationTestUtils.generate_run_date(_run_date_type, _custom_run_date)
-
- BaseSurveyTestSuite.SurveyTestSuite.TEST_JOB_DIR = "SURVEY_" + BaseSurveyTestSuite.SurveyTestSuite.TEST_START_DATE
-
- # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
- sys.argv[1:] = _args.unittest_args
- tests: TestSuite = TestLoader().loadTestsFromTestCase(FullTestSurvey)
-
- if _test_report_dir is None:
- _test_report_dir = BaseSurveyTestSuite.SurveyTestSuite.TEST_JOB_DIR
-
- runner = HTMLTestRunner(output=_test_report_dir, combine_reports = True)
- runner.run(tests)
diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py
index 4088edf..566b2f0 100644
--- a/tests/integration/partial/integration_test_utils.py
+++ b/tests/integration/partial/integration_test_utils.py
@@ -58,8 +58,8 @@ class IntegrationTestUtils:
IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH = _config_file
nowstring: str = IntegrationTestUtils.get_now_string()
- prefix: str = f"temp_{test_prefix}_" + nowstring
- # prefix: str = f"temp_{test_prefix}"
+ # prefix: str = f"temp_{test_prefix}_" + nowstring
+ prefix: str = f"temp_{test_prefix}"
IntegrationTestUtils.TEST_OUT_PATH = _outdir + prefix + os.sep
IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path
diff --git a/tests/integration/test_suites/advisory_test_suite.py b/tests/integration/test_suites/advisory_test_suite.py
index afd3fbf..abe06ab 100644
--- a/tests/integration/test_suites/advisory_test_suite.py
+++ b/tests/integration/test_suites/advisory_test_suite.py
@@ -15,7 +15,7 @@ class BaseAdvisoryTestSuite:
"""
ADVISORY_COMPONENT_NAME = "Advisory"
"""
- the prefix of the directory created by thr Processor.py, e.g. DEPOSITION_2020-01-01
+ the prefix of the directory created by thr Processor.py, e.g. ADVISORY_2020-01-01
"""
ADVISORY_PROCESSOR_DIR = "ADVISORY"
diff --git a/tests/integration/test_suites/epi_test_suite.py b/tests/integration/test_suites/epi_test_suite.py
index e0342bb..c931d7a 100644
--- a/tests/integration/test_suites/epi_test_suite.py
+++ b/tests/integration/test_suites/epi_test_suite.py
@@ -9,24 +9,29 @@ class BaseEpiTestSuite:
class EpiTestSuite(unittest.TestCase):
- TEST_OUT_PATH: str = None
- TEST_START_DATE: str = None
- TEST_JOB_DIR: str = None
+ """
+ the name passed to the Processor.py to define the component
+ """
+ EPI_COMPONENT_NAME = "Epidemiology"
+ """
+ the prefix of the directory created by thr Processor.py, e.g. EPI_2020-01-01
+ """
+ EPI_PROCESSOR_DIR = "EPI"
@abc.abstractmethod
def set_expected_values(self):
pass
def test_standard_run_input_status_success(self):
- status_file_path = os.path.join(BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH,
- BaseEpiTestSuite.EpiTestSuite.TEST_JOB_DIR, "STATUS_SUCCESS")
+ status_file_path = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
+ IntegrationTestUtils.TEST_JOB_DIR, "STATUS_SUCCESS")
success_file_exists: bool = os.path.isfile(status_file_path)
self.assertTrue(success_file_exists)
def test_standard_run_input_all_stem_rust_files_produced(self):
- alpha_beta_gamma_root = os.path.join(BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH,
- BaseEpiTestSuite.EpiTestSuite.TEST_JOB_DIR,
+ alpha_beta_gamma_root = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
+ IntegrationTestUtils.TEST_JOB_DIR,
"EastAfrica",
"StemRust",
"*psbeta0.004gamma0.00025alpha1.0")
@@ -37,8 +42,8 @@ class BaseEpiTestSuite:
self.check_wildcard_exists_and_not_empty(alpha_beta_gamma_root, ".png")
self.check_wildcard_exists_and_not_empty(alpha_beta_gamma_root, ".tif")
- env_root = os.path.join(BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH,
- BaseEpiTestSuite.EpiTestSuite.TEST_JOB_DIR,
+ env_root = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
+ IntegrationTestUtils.TEST_JOB_DIR,
"EastAfrica",
"StemRust",
"infections_temp_config_Epidemiology")
@@ -47,8 +52,8 @@ class BaseEpiTestSuite:
self.check_wildcard_exists_and_not_empty(env_root, "*env.png")
self.check_wildcard_exists_and_not_empty(env_root, "*env.tif")
- data_input_root = os.path.join(BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH,
- BaseEpiTestSuite.EpiTestSuite.TEST_JOB_DIR,
+ data_input_root = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
+ IntegrationTestUtils.TEST_JOB_DIR,
"EastAfrica",
"StemRust",
"input_data",
@@ -57,8 +62,8 @@ class BaseEpiTestSuite:
self.check_wildcard_exists_and_not_empty(data_input_root, "*deposition.csv")
self.check_wildcard_exists_and_not_empty(data_input_root, "*environment.csv")
- map_spam_root = os.path.join(BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH,
- BaseEpiTestSuite.EpiTestSuite.TEST_JOB_DIR,
+ map_spam_root = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
+ IntegrationTestUtils.TEST_JOB_DIR,
"EastAfrica",
"StemRust",
"input_data",
@@ -69,16 +74,16 @@ class BaseEpiTestSuite:
def test_standard_run_input_all_plotting_files_produced(self):
- input_csvs_root = os.path.join(BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH,
- BaseEpiTestSuite.EpiTestSuite.TEST_JOB_DIR,
+ input_csvs_root = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
+ IntegrationTestUtils.TEST_JOB_DIR,
"plotting",
"input_csvs",
"*stem*")
self.check_wildcard_exists_and_not_empty(input_csvs_root, ".csv")
- images_root = os.path.join(BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH,
- BaseEpiTestSuite.EpiTestSuite.TEST_JOB_DIR,
+ images_root = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
+ IntegrationTestUtils.TEST_JOB_DIR,
"plotting",
"images",
"*stem*")
diff --git a/tests/integration/test_suites/survey_test_suite.py b/tests/integration/test_suites/survey_test_suite.py
index 6c6804d..83ef237 100644
--- a/tests/integration/test_suites/survey_test_suite.py
+++ b/tests/integration/test_suites/survey_test_suite.py
@@ -9,6 +9,15 @@ class BaseSurveyTestSuite:
class SurveyTestSuite(unittest.TestCase):
+ """
+ the name passed to the Processor.py to define the component
+ """
+ SURVEY_COMPONENT_NAME = "Survey"
+ """
+ the prefix of the directory created by thr Processor.py, e.g. SURVEY_2020-01-01
+ """
+ SURVEY_PROCESSOR_DIR = "SURVEYDATA"
+
@abc.abstractmethod
def set_expected_values(self):
pass
--
GitLab