diff --git a/tests/integration/full/full_test_env_suit.py b/tests/integration/full/full_test_env_suit.py
index ba9c0e73833efdcc2427ad6cb9e3341753886271..a8537e3b80a0859b255d798a6c7a7952353cbc22 100644
--- a/tests/integration/full/full_test_env_suit.py
+++ b/tests/integration/full/full_test_env_suit.py
@@ -31,6 +31,11 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
@staticmethod
def write_temp_run_config_file():
+ """
+ IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH will have been set from the command line args, and point to the
+ production sys config file. We need to modify this file to point to the test output directory.
+ :return:
+ """
sys_config = IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH
sys_config_dict: dict = parse_json_file_with_tokens(sys_config)
sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
@@ -44,7 +49,7 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = job_sys_config_file
run_config = IntegrationTestUtils.RUN_CONFIG_FILE_PATH
- run_config_temp_dict: dict = IntegrationTestUtils.load_json_file(run_config)
+ run_config_temp_dict: dict = parse_json_file_with_tokens(run_config, sys_config_dict)
run_dict: dict = copy.deepcopy(run_config_temp_dict)
run_dict['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
run_dict['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
@@ -52,7 +57,12 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
run_dict['EWS-Plotting']['Ethiopia']['FilterForCountry'] = "True"
run_dict['EWS-Plotting']['Kenya']['FilterForCountry'] = "True"
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)
+ job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_dict, job_run_config_file)
+
+ # set the run config file path to the new job-specific file
+ IntegrationTestUtils.RUN_CONFIG_FILE_PATH = job_run_config_file
+
@staticmethod
diff --git a/tests/integration/full/full_test_epi.py b/tests/integration/full/full_test_epi.py
index 7a4063c30e272ca5d56f72c24542b401c9a0a2d5..a4a8faa48dd413842b670b1bd24577fd718f3513 100644
--- a/tests/integration/full/full_test_epi.py
+++ b/tests/integration/full/full_test_epi.py
@@ -5,6 +5,7 @@ import sys
from ews.coordinator.processor_deposition import ProcessorDeposition
from ews.coordinator.processor_environment import ProcessorEnvironment
from ews.coordinator.processor_epidemiology import ProcessorEpidemiology
+from ews.coordinator.utils.processor_utils import parse_json_file_with_tokens
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.depo_test_suite import BaseDepoTestSuite
from integration.test_suites.env_suit_test_suite import BaseEnvSuitTestSuite
@@ -34,23 +35,49 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
@staticmethod
def write_temp_run_config_files():
- default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
- default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
- run_dict: dict = copy.deepcopy(default_config_dict)
- run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
+ # default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
+ # default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
+ # run_dict: dict = copy.deepcopy(default_config_dict)
+ #
+ #
+ # # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
+ # os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
+ # IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ # IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)
+
+ """
+ IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH will have been set from the command line args, and point to the
+ production sys config file. We need to modify this file to point to the test output directory.
+ :return:
+ """
+ sys_config = IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH
+ sys_config_dict: dict = parse_json_file_with_tokens(sys_config)
+
+ sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
+ sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
+
+ os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
+ job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+ IntegrationTestUtils.write_json_file(sys_config_dict, job_sys_config_file)
+
+ # now set the sys config file path to the new job-specific file
+ IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = job_sys_config_file
+
+ run_config = IntegrationTestUtils.RUN_CONFIG_FILE_PATH
+ run_config_temp_dict: dict = parse_json_file_with_tokens(run_config, sys_config_dict)
+ run_dict: dict = copy.deepcopy(run_config_temp_dict)
# we need to run the env suit pipeline as well
run_dict['Environment']['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
run_dict['Environment']['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
run_dict['Environment']['OUTPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
- #make sure we activate server download
+ # make sure we activate server download
run_dict['Survey']['SkipServerDownload'] = False
# change the config for the Epi so it only runs on a single past date. Requires use of a different
# FileListerFunction and a different calculation span
- run_dict['Epidemiology']['CalculationSpanDays'] = [0, 1]
- run_dict['Epidemiology']['Deposition']['FileListerFunction'] = "list_onefile_operational"
- run_dict['Epidemiology']['Environment']['FileListerFunction'] = "list_onefile_operational"
+ run_dict['CalculationSpanDays'] = [0, 1]
+ run_dict['Deposition']['FileListerFunction'] = "list_onefile_operational"
+ run_dict['Environment']['FileListerFunction'] = "list_onefile_operational"
"""
When set to true, continue mode speeds up computation time and reduces data volume by starting from the start of
@@ -58,14 +85,15 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
inputs specified alongside Deposition and Environment. In this test version, we do not have prior runs to draw
from, so setting to false
"""
- run_dict['Epidemiology']['continue'] = False
+ run_dict['continue'] = False
previous_day_string: str = IntegrationTestUtils.get_day_before_as_string(IntegrationTestUtils.TEST_START_DATE)
- run_dict['Survey']['SeasonStartString'] = previous_day_string
+ run_dict['SeasonStartString'] = previous_day_string
- # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
- os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)
+ job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_dict, job_run_config_file)
+
+ # set the run config file path to the new job-specific file
+ IntegrationTestUtils.RUN_CONFIG_FILE_PATH = job_run_config_file
@staticmethod
diff --git a/tests/integration/full/full_test_survey.py b/tests/integration/full/full_test_survey.py
index bd2b23050a747f4797e2d1702a180d009c6a73aa..fdb71f59d6e37b1034bab173afa62c305596a2e7 100644
--- a/tests/integration/full/full_test_survey.py
+++ b/tests/integration/full/full_test_survey.py
@@ -3,6 +3,7 @@ import os
import sys
from ews.coordinator.processor_surveys import ProcessorSurveys
+from ews.coordinator.utils.processor_utils import parse_json_file_with_tokens
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.survey_test_suite import BaseSurveyTestSuite
@@ -28,22 +29,41 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
@staticmethod
def write_temp_run_config_file():
- default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
- default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
- run_dict: dict = copy.deepcopy(default_config_dict)
- run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['Survey']['SkipServerDownload'] = False
+
+ """
+ IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH will have been set from the command line args, and point to the
+ production sys config file. We need to modify this file to point to the test output directory.
+ :return:
+ """
+ sys_config = IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH
+ sys_config_dict: dict = parse_json_file_with_tokens(sys_config)
+ sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
+ sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)
+ job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+ IntegrationTestUtils.write_json_file(sys_config_dict, job_sys_config_file)
+
+ # now set the sys config file path to the new job-specific file
+ IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = job_sys_config_file
+
+ run_config = IntegrationTestUtils.RUN_CONFIG_FILE_PATH
+ run_config_temp_dict: dict = parse_json_file_with_tokens(run_config, sys_config_dict)
+ run_dict: dict = copy.deepcopy(run_config_temp_dict)
+ run_dict['SkipServerDownload'] = False
+
+ job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_dict, job_run_config_file)
+
+ # set the run config file path to the new job-specific file
+ IntegrationTestUtils.RUN_CONFIG_FILE_PATH = job_run_config_file
+
@staticmethod
def run_survey_pipeline():
survey_processor = ProcessorSurveys()
IntegrationTestUtils.run_external_pipeline(BaseSurveyTestSuite.SurveyTestSuite.SURVEY_COMPONENT_NAME,
- BaseSurveyTestSuite.SurveyTestSuite.SURVEY_PROCESSOR_DIR,
+ BaseSurveyTestSuite.SurveyTestSuite.SURVEY_PROCESSOR_DIR,
IntegrationTestUtils.TEST_START_DATE,
survey_processor)