From c8991098e1d2ba6d89b8286399eb42194f000ef4 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Fri, 4 Nov 2022 13:53:21 +0000
Subject: [PATCH] restoring the original sys.exit to ProcessorUtils, dealing
with exit commands in the unit tests by catching them
---
coordinator/ProcessorUtils.py | 1 -
tests/integration/integration_test_utils.py | 33 +++++++++++++++++++++
tests/integration/test_advisory.py | 31 ++++++++-----------
tests/integration/test_deposition.py | 20 +++----------
tests/integration/test_env_suit.py | 23 ++++----------
tests/integration/test_survey.py | 25 ++++++----------
6 files changed, 65 insertions(+), 68 deletions(-)
diff --git a/coordinator/ProcessorUtils.py b/coordinator/ProcessorUtils.py
index ea8fe8f..8ddfa4b 100644
--- a/coordinator/ProcessorUtils.py
+++ b/coordinator/ProcessorUtils.py
@@ -169,7 +169,6 @@ def endScript(premature=True):
logger.info(f'--------')
- # if __name__ == '__main__':
sys.exit()
def endJob(status,ignore_inprogress=False,**kwargs):
diff --git a/tests/integration/integration_test_utils.py b/tests/integration/integration_test_utils.py
index e524461..03a616b 100644
--- a/tests/integration/integration_test_utils.py
+++ b/tests/integration/integration_test_utils.py
@@ -7,6 +7,7 @@ from zipfile import ZipFile
class IntegrationTestUtils:
+ EMAIL_CRED_PATH: str = "../test_data/test_deployment/envs/Cred_gmail.json"
DEFAULT_CONFIG_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json"
TEMP_CONFIG_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/workspace/temp_config.json"
EXAMPLE_SURVEY_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_survey_run.zip"
@@ -47,3 +48,35 @@ class IntegrationTestUtils:
@staticmethod
def check_file_not_empty(file_path: str):
return os.stat(file_path).st_size != 0
+
+
+ @staticmethod
+ def run_pipeline(component: str,
+ start_date: str,
+ **kwargs):
+
+ # need EMAIL_CRED in the environment before we import Processor
+ os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
+ from Processor import run_Process, set_log_level
+
+ args_dict: dict = {}
+
+ # note, possible to override these values in the kwargs loop below
+ args_dict['live'] = False
+ args_dict['noupload'] = True
+ args_dict['start_date'] = start_date
+ args_dict['component'] = component
+ args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH]
+ args_dict['log_level'] = 'info'
+
+ for key, value in kwargs.items():
+ args_dict[key] = value
+
+ log_level = args_dict['log_level']
+ set_log_level(log_level)
+
+ try:
+ run_Process(args_dict)
+ except SystemExit:
+ # we will eventually want to throw these to the calling class to be dealt with
+ pass
diff --git a/tests/integration/test_advisory.py b/tests/integration/test_advisory.py
index ea238a9..cfa8d72 100644
--- a/tests/integration/test_advisory.py
+++ b/tests/integration/test_advisory.py
@@ -8,17 +8,18 @@ from integration.integration_test_utils import IntegrationTestUtils
class TestAdvisory(unittest.TestCase):
TEST_OUT_PATH: str = "not_set"
- TEST_RUN_DATE: str = '20221001'
- TEST_JOB_DIR: str = "SUMMARY_" + TEST_RUN_DATE
+ TEST_START_DATE: str = '20221001'
+ TEST_JOB_DIR: str = "SUMMARY_" + TEST_START_DATE
@classmethod
def setUpClass(cls) -> None:
- TestAdvisory.write_temp_run_config_file()
+ TestAdvisory.write_temp_run_config_files()
+ TestAdvisory.unpack_dependencies()
TestAdvisory.run_advisory_pipeline()
@staticmethod
- def write_temp_run_config_file():
+ def write_temp_run_config_files():
nowstring: str = IntegrationTestUtils.get_now_string()
prefix: str = "temp_advisory_" + nowstring
# prefix: str = ""
@@ -33,25 +34,19 @@ class TestAdvisory(unittest.TestCase):
IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
- @staticmethod
- def run_advisory_pipeline():
-
- os.environ["EMAIL_CRED"] = "../test_data/test_deployment/envs/Cred_gmail.json"
- from Processor import run_Process, set_log_level
- args_dict: dict = {}
- args_dict['component'] = 'Advisory'
- args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH]
- args_dict['log_level'] = 'info'
- args_dict['live'] = False
- args_dict['start_date'] = TestAdvisory.TEST_RUN_DATE
- args_dict['noupload'] = True
- set_log_level(args_dict['log_level'])
+ @staticmethod
+ def unpack_dependencies():
IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_SURVEY_FILE_PATH, TestAdvisory.TEST_OUT_PATH)
IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_DEPO_FILE_PATH, TestAdvisory.TEST_OUT_PATH)
IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_ENV_SUIT_FILE_PATH, TestAdvisory.TEST_OUT_PATH)
- run_Process(args_dict)
+
+ @staticmethod
+ def run_advisory_pipeline():
+ component = 'Advisory'
+ IntegrationTestUtils.run_pipeline(component, TestAdvisory.TEST_START_DATE)
+
def test_standard_run_input_status_success(self):
status_file_path = os.path.join(TestAdvisory.TEST_OUT_PATH, TestAdvisory.TEST_JOB_DIR, "STATUS_SUCCESS")
diff --git a/tests/integration/test_deposition.py b/tests/integration/test_deposition.py
index 6316839..f12c387 100644
--- a/tests/integration/test_deposition.py
+++ b/tests/integration/test_deposition.py
@@ -7,8 +7,8 @@ from integration.integration_test_utils import IntegrationTestUtils
class TestDeposition(unittest.TestCase):
TEST_OUT_PATH: str = "not_set"
- TEST_RUN_DATE: str = '20221001'
- TEST_JOB_DIR: str = "DEPOSITION_" + TEST_RUN_DATE
+ TEST_START_DATE: str = '20221001'
+ TEST_JOB_DIR: str = "DEPOSITION_" + TEST_START_DATE
@classmethod
@@ -38,20 +38,8 @@ class TestDeposition(unittest.TestCase):
@staticmethod
def run_depo_pipeline():
-
- os.environ["EMAIL_CRED"] = "../test_data/test_deployment/envs/Cred_gmail.json"
- from Processor import run_Process, set_log_level
-
- args_dict: dict = {}
- args_dict['component'] = 'Deposition'
- args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH]
- args_dict['log_level'] = 'error'
- args_dict['live'] = False
- args_dict['start_date'] = TestDeposition.TEST_RUN_DATE
- args_dict['noupload'] = True
- set_log_level(args_dict['log_level'])
-
- run_Process(args_dict)
+ component = 'Deposition'
+ IntegrationTestUtils.run_pipeline(component, TestDeposition.TEST_START_DATE)
def test_standard_run_input_status_success(self):
diff --git a/tests/integration/test_env_suit.py b/tests/integration/test_env_suit.py
index b3a46aa..b51487f 100644
--- a/tests/integration/test_env_suit.py
+++ b/tests/integration/test_env_suit.py
@@ -8,8 +8,8 @@ from integration.integration_test_utils import IntegrationTestUtils
class TestEnvSuit(unittest.TestCase):
TEST_OUT_PATH: str = "not_set"
- TEST_RUN_DATE: str = '20221001'
- TEST_JOB_DIR: str = "ENVIRONMENT_2.0_" + TEST_RUN_DATE
+ TEST_START_DATE: str = '20221001'
+ TEST_JOB_DIR: str = "ENVIRONMENT_2.0_" + TEST_START_DATE
@classmethod
def setUpClass(cls) -> None:
@@ -20,8 +20,8 @@ class TestEnvSuit(unittest.TestCase):
@staticmethod
def write_temp_run_config_file():
nowstring: str = IntegrationTestUtils.get_now_string()
- # prefix: str = "temp_depo_" + nowstring
- prefix: str = ""
+ prefix: str = "temp_depo_" + nowstring
+ # prefix: str = ""
default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
@@ -41,19 +41,8 @@ class TestEnvSuit(unittest.TestCase):
@staticmethod
def run_env_pipeline():
- os.environ["EMAIL_CRED"] = "../test_data/test_deployment/envs/Cred_gmail.json"
- from Processor import run_Process, set_log_level
-
- args_dict: dict = {}
- args_dict['component'] = 'Environment'
- args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH]
- args_dict['log_level'] = 'error'
- args_dict['live'] = False
- args_dict['start_date'] = TestEnvSuit.TEST_RUN_DATE
- args_dict['noupload'] = True
- set_log_level(args_dict['log_level'])
-
- run_Process(args_dict)
+ component = 'Environment'
+ IntegrationTestUtils.run_pipeline(component, TestEnvSuit.TEST_START_DATE)
def test_standard_run_input_status_success(self):
diff --git a/tests/integration/test_survey.py b/tests/integration/test_survey.py
index 0dbb692..00f1215 100644
--- a/tests/integration/test_survey.py
+++ b/tests/integration/test_survey.py
@@ -8,12 +8,13 @@ from integration.integration_test_utils import IntegrationTestUtils
class TestSurvey(unittest.TestCase):
TEST_OUT_PATH: str = "not_set"
- TEST_RUN_DATE: str = '20221001'
- TEST_JOB_DIR: str = "SURVEYDATA_" + TEST_RUN_DATE
+ TEST_START_DATE: str = '20221001'
+ TEST_JOB_DIR: str = "SURVEYDATA_" + TEST_START_DATE
@classmethod
def setUpClass(cls) -> None:
TestSurvey.write_temp_run_config_file()
+ TestSurvey.unpack_dependencies()
TestSurvey.run_survey_pipeline()
@@ -35,22 +36,14 @@ class TestSurvey(unittest.TestCase):
IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
@staticmethod
- def run_survey_pipeline():
-
- os.environ["EMAIL_CRED"] = "../test_data/test_deployment/envs/Cred_gmail.json"
- from Processor import run_Process, set_log_level
- args_dict: dict = {}
- args_dict['component'] = 'Survey'
- args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH]
- args_dict['log_level'] = 'info'
- args_dict['live'] = False
- args_dict['start_date'] = TestSurvey.TEST_RUN_DATE
- args_dict['noupload'] = True
- set_log_level(args_dict['log_level'])
-
+ def unpack_dependencies():
IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_SURVEY_FILE_PATH, TestSurvey.TEST_OUT_PATH)
- run_Process(args_dict)
+ @staticmethod
+ def run_survey_pipeline():
+ component = 'Survey'
+ IntegrationTestUtils.run_pipeline(component, TestSurvey.TEST_START_DATE)
+
def test_standard_run_input_status_success(self):
status_file_path = os.path.join(TestSurvey.TEST_OUT_PATH, TestSurvey.TEST_JOB_DIR, "STATUS_SUCCESS")
--
GitLab