diff --git a/coordinator/ProcessorUtils.py b/coordinator/ProcessorUtils.py
index ea8fe8f88d92f05646f5df0aa89546311f669e8f..8ddfa4b0e974e3e9f88cc4b247c61c6b1f6e614d 100644
--- a/coordinator/ProcessorUtils.py
+++ b/coordinator/ProcessorUtils.py
@@ -169,7 +169,6 @@ def endScript(premature=True):
logger.info(f'--------')
- # if __name__ == '__main__':
sys.exit()
def endJob(status,ignore_inprogress=False,**kwargs):
diff --git a/tests/integration/integration_test_utils.py b/tests/integration/integration_test_utils.py
index e5244614a516f52f626f2f5fd2274f7438fb4781..03a616b393bf770b168230ce14a0f3aaff197741 100644
--- a/tests/integration/integration_test_utils.py
+++ b/tests/integration/integration_test_utils.py
@@ -7,6 +7,7 @@ from zipfile import ZipFile
class IntegrationTestUtils:
+ EMAIL_CRED_PATH: str = "../test_data/test_deployment/envs/Cred_gmail.json"
DEFAULT_CONFIG_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json"
TEMP_CONFIG_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/workspace/temp_config.json"
EXAMPLE_SURVEY_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_survey_run.zip"
@@ -47,3 +48,35 @@ class IntegrationTestUtils:
@staticmethod
def check_file_not_empty(file_path: str):
return os.stat(file_path).st_size != 0
+
+
+ @staticmethod
+ def run_pipeline(component: str,
+ start_date: str,
+ **kwargs):
+
+ # need EMAIL_CRED in the environment before we import Processor
+ os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
+ from Processor import run_Process, set_log_level
+
+ args_dict: dict = {}
+
+ # note, possible to override these values in the kwargs loop below
+ args_dict['live'] = False
+ args_dict['noupload'] = True
+ args_dict['start_date'] = start_date
+ args_dict['component'] = component
+ args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH]
+ args_dict['log_level'] = 'info'
+
+ for key, value in kwargs.items():
+ args_dict[key] = value
+
+ log_level = args_dict['log_level']
+ set_log_level(log_level)
+
+ try:
+ run_Process(args_dict)
+ except SystemExit:
+ # we will eventually want to throw these to the calling class to be dealt with
+ pass
diff --git a/tests/integration/test_advisory.py b/tests/integration/test_advisory.py
index ea238a9be6fb26f479e30f7122e1f75b2c64c87a..cfa8d7240f0e3f48e17cbf664ebf6e23e1c628ac 100644
--- a/tests/integration/test_advisory.py
+++ b/tests/integration/test_advisory.py
@@ -8,17 +8,18 @@ from integration.integration_test_utils import IntegrationTestUtils
class TestAdvisory(unittest.TestCase):
TEST_OUT_PATH: str = "not_set"
- TEST_RUN_DATE: str = '20221001'
- TEST_JOB_DIR: str = "SUMMARY_" + TEST_RUN_DATE
+ TEST_START_DATE: str = '20221001'
+ TEST_JOB_DIR: str = "SUMMARY_" + TEST_START_DATE
@classmethod
def setUpClass(cls) -> None:
- TestAdvisory.write_temp_run_config_file()
+ TestAdvisory.write_temp_run_config_files()
+ TestAdvisory.unpack_dependencies()
TestAdvisory.run_advisory_pipeline()
@staticmethod
- def write_temp_run_config_file():
+ def write_temp_run_config_files():
nowstring: str = IntegrationTestUtils.get_now_string()
prefix: str = "temp_advisory_" + nowstring
# prefix: str = ""
@@ -33,25 +34,19 @@ class TestAdvisory(unittest.TestCase):
IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
- @staticmethod
- def run_advisory_pipeline():
-
- os.environ["EMAIL_CRED"] = "../test_data/test_deployment/envs/Cred_gmail.json"
- from Processor import run_Process, set_log_level
- args_dict: dict = {}
- args_dict['component'] = 'Advisory'
- args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH]
- args_dict['log_level'] = 'info'
- args_dict['live'] = False
- args_dict['start_date'] = TestAdvisory.TEST_RUN_DATE
- args_dict['noupload'] = True
- set_log_level(args_dict['log_level'])
+ @staticmethod
+ def unpack_dependencies():
IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_SURVEY_FILE_PATH, TestAdvisory.TEST_OUT_PATH)
IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_DEPO_FILE_PATH, TestAdvisory.TEST_OUT_PATH)
IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_ENV_SUIT_FILE_PATH, TestAdvisory.TEST_OUT_PATH)
- run_Process(args_dict)
+
+ @staticmethod
+ def run_advisory_pipeline():
+ component = 'Advisory'
+ IntegrationTestUtils.run_pipeline(component, TestAdvisory.TEST_START_DATE)
+
def test_standard_run_input_status_success(self):
status_file_path = os.path.join(TestAdvisory.TEST_OUT_PATH, TestAdvisory.TEST_JOB_DIR, "STATUS_SUCCESS")
diff --git a/tests/integration/test_deposition.py b/tests/integration/test_deposition.py
index 6316839a9dc291ac9077287676e642c79f86719c..f12c387890adae0b581dc495b989850b579d2f31 100644
--- a/tests/integration/test_deposition.py
+++ b/tests/integration/test_deposition.py
@@ -7,8 +7,8 @@ from integration.integration_test_utils import IntegrationTestUtils
class TestDeposition(unittest.TestCase):
TEST_OUT_PATH: str = "not_set"
- TEST_RUN_DATE: str = '20221001'
- TEST_JOB_DIR: str = "DEPOSITION_" + TEST_RUN_DATE
+ TEST_START_DATE: str = '20221001'
+ TEST_JOB_DIR: str = "DEPOSITION_" + TEST_START_DATE
@classmethod
@@ -38,20 +38,8 @@ class TestDeposition(unittest.TestCase):
@staticmethod
def run_depo_pipeline():
-
- os.environ["EMAIL_CRED"] = "../test_data/test_deployment/envs/Cred_gmail.json"
- from Processor import run_Process, set_log_level
-
- args_dict: dict = {}
- args_dict['component'] = 'Deposition'
- args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH]
- args_dict['log_level'] = 'error'
- args_dict['live'] = False
- args_dict['start_date'] = TestDeposition.TEST_RUN_DATE
- args_dict['noupload'] = True
- set_log_level(args_dict['log_level'])
-
- run_Process(args_dict)
+ component = 'Deposition'
+ IntegrationTestUtils.run_pipeline(component, TestDeposition.TEST_START_DATE)
def test_standard_run_input_status_success(self):
diff --git a/tests/integration/test_env_suit.py b/tests/integration/test_env_suit.py
index b3a46aa47086cfa76270aa7347c2d649d8d54b0f..b51487f95d455b64ae5ba871c828f6089ca7956c 100644
--- a/tests/integration/test_env_suit.py
+++ b/tests/integration/test_env_suit.py
@@ -8,8 +8,8 @@ from integration.integration_test_utils import IntegrationTestUtils
class TestEnvSuit(unittest.TestCase):
TEST_OUT_PATH: str = "not_set"
- TEST_RUN_DATE: str = '20221001'
- TEST_JOB_DIR: str = "ENVIRONMENT_2.0_" + TEST_RUN_DATE
+ TEST_START_DATE: str = '20221001'
+ TEST_JOB_DIR: str = "ENVIRONMENT_2.0_" + TEST_START_DATE
@classmethod
def setUpClass(cls) -> None:
@@ -20,8 +20,8 @@ class TestEnvSuit(unittest.TestCase):
@staticmethod
def write_temp_run_config_file():
nowstring: str = IntegrationTestUtils.get_now_string()
- # prefix: str = "temp_depo_" + nowstring
- prefix: str = ""
+ prefix: str = "temp_depo_" + nowstring
+ # prefix: str = ""
default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
@@ -41,19 +41,8 @@ class TestEnvSuit(unittest.TestCase):
@staticmethod
def run_env_pipeline():
- os.environ["EMAIL_CRED"] = "../test_data/test_deployment/envs/Cred_gmail.json"
- from Processor import run_Process, set_log_level
-
- args_dict: dict = {}
- args_dict['component'] = 'Environment'
- args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH]
- args_dict['log_level'] = 'error'
- args_dict['live'] = False
- args_dict['start_date'] = TestEnvSuit.TEST_RUN_DATE
- args_dict['noupload'] = True
- set_log_level(args_dict['log_level'])
-
- run_Process(args_dict)
+ component = 'Environment'
+ IntegrationTestUtils.run_pipeline(component, TestEnvSuit.TEST_START_DATE)
def test_standard_run_input_status_success(self):
diff --git a/tests/integration/test_survey.py b/tests/integration/test_survey.py
index 0dbb692ad9c4027a6466a6dbaf620dd03ad6c2d3..00f12154180573d27daf93e72c57dd911132a1a7 100644
--- a/tests/integration/test_survey.py
+++ b/tests/integration/test_survey.py
@@ -8,12 +8,13 @@ from integration.integration_test_utils import IntegrationTestUtils
class TestSurvey(unittest.TestCase):
TEST_OUT_PATH: str = "not_set"
- TEST_RUN_DATE: str = '20221001'
- TEST_JOB_DIR: str = "SURVEYDATA_" + TEST_RUN_DATE
+ TEST_START_DATE: str = '20221001'
+ TEST_JOB_DIR: str = "SURVEYDATA_" + TEST_START_DATE
@classmethod
def setUpClass(cls) -> None:
TestSurvey.write_temp_run_config_file()
+ TestSurvey.unpack_dependencies()
TestSurvey.run_survey_pipeline()
@@ -35,22 +36,14 @@ class TestSurvey(unittest.TestCase):
IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
@staticmethod
- def run_survey_pipeline():
-
- os.environ["EMAIL_CRED"] = "../test_data/test_deployment/envs/Cred_gmail.json"
- from Processor import run_Process, set_log_level
- args_dict: dict = {}
- args_dict['component'] = 'Survey'
- args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH]
- args_dict['log_level'] = 'info'
- args_dict['live'] = False
- args_dict['start_date'] = TestSurvey.TEST_RUN_DATE
- args_dict['noupload'] = True
- set_log_level(args_dict['log_level'])
-
+ def unpack_dependencies():
IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_SURVEY_FILE_PATH, TestSurvey.TEST_OUT_PATH)
- run_Process(args_dict)
+ @staticmethod
+ def run_survey_pipeline():
+ component = 'Survey'
+ IntegrationTestUtils.run_pipeline(component, TestSurvey.TEST_START_DATE)
+
def test_standard_run_input_status_success(self):
status_file_path = os.path.join(TestSurvey.TEST_OUT_PATH, TestSurvey.TEST_JOB_DIR, "STATUS_SUCCESS")