reverting start date for more testing

tests/integration/full/full_test_env_suit.py

@@ -31,6 +31,11 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
     @staticmethod
     def write_temp_run_config_file():
 
+        """
+        IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH will have been set from the command line args, and point to the
+        production sys config file. We need to modify this file to point to the test output directory.
+        :return:
+        """
         sys_config = IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH
         sys_config_dict: dict = parse_json_file_with_tokens(sys_config)
         sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
@@ -44,7 +49,7 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
         IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = job_sys_config_file
 
         run_config = IntegrationTestUtils.RUN_CONFIG_FILE_PATH
-        run_config_temp_dict: dict = IntegrationTestUtils.load_json_file(run_config)
+        run_config_temp_dict: dict = parse_json_file_with_tokens(run_config, sys_config_dict)
         run_dict: dict = copy.deepcopy(run_config_temp_dict)
         run_dict['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
         run_dict['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
@@ -52,7 +57,12 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
         run_dict['EWS-Plotting']['Ethiopia']['FilterForCountry'] = "True"
         run_dict['EWS-Plotting']['Kenya']['FilterForCountry'] = "True"
 
-        IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)
+        job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+        IntegrationTestUtils.write_json_file(run_dict, job_run_config_file)
+
+        # set the run config file path to the new job-specific file
+        IntegrationTestUtils.RUN_CONFIG_FILE_PATH = job_run_config_file
+
 
 
     @staticmethod

tests/integration/full/full_test_epi.py

@@ -5,6 +5,7 @@ import sys
 from ews.coordinator.processor_deposition import ProcessorDeposition
 from ews.coordinator.processor_environment import ProcessorEnvironment
 from ews.coordinator.processor_epidemiology import ProcessorEpidemiology
+from ews.coordinator.utils.processor_utils import parse_json_file_with_tokens
 from integration.partial.integration_test_utils import IntegrationTestUtils
 from integration.test_suites.depo_test_suite import BaseDepoTestSuite
 from integration.test_suites.env_suit_test_suite import BaseEnvSuitTestSuite
@@ -34,23 +35,49 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
 
     @staticmethod
     def write_temp_run_config_files():
-        default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
-        default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
-        run_dict: dict = copy.deepcopy(default_config_dict)
-        run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
-        run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
+        # default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
+        # default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
+        # run_dict: dict = copy.deepcopy(default_config_dict)
+        #
+        #
+        # # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
+        # os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
+        # IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+        # IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)
+
+        """
+        IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH will have been set from the command line args, and point to the
+        production sys config file. We need to modify this file to point to the test output directory.
+        :return:
+        """
+        sys_config = IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH
+        sys_config_dict: dict = parse_json_file_with_tokens(sys_config)
+
+        sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
+        sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
+
+        os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
+        job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+        IntegrationTestUtils.write_json_file(sys_config_dict, job_sys_config_file)
+
+        # now set the sys config file path to the new job-specific file
+        IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = job_sys_config_file
+
+        run_config = IntegrationTestUtils.RUN_CONFIG_FILE_PATH
+        run_config_temp_dict: dict = parse_json_file_with_tokens(run_config, sys_config_dict)
+        run_dict: dict = copy.deepcopy(run_config_temp_dict)
         # we need to run the env suit pipeline as well
         run_dict['Environment']['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
         run_dict['Environment']['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
         run_dict['Environment']['OUTPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
-        #make sure we activate server download
+        # make sure we activate server download
         run_dict['Survey']['SkipServerDownload'] = False
 
         # change the config for the Epi so it only runs on a single past date. Requires use of a different
         # FileListerFunction and a different calculation span
-        run_dict['Epidemiology']['CalculationSpanDays'] = [0, 1]
-        run_dict['Epidemiology']['Deposition']['FileListerFunction'] = "list_onefile_operational"
-        run_dict['Epidemiology']['Environment']['FileListerFunction'] = "list_onefile_operational"
+        run_dict['CalculationSpanDays'] = [0, 1]
+        run_dict['Deposition']['FileListerFunction'] = "list_onefile_operational"
+        run_dict['Environment']['FileListerFunction'] = "list_onefile_operational"
 
         """
         When set to true, continue mode speeds up computation time and reduces data volume by starting from the start of 
@@ -58,14 +85,15 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
         inputs specified alongside Deposition and Environment. In this test version, we do not have prior runs to draw 
         from, so setting to false
         """
-        run_dict['Epidemiology']['continue'] = False
+        run_dict['continue'] = False
         previous_day_string: str = IntegrationTestUtils.get_day_before_as_string(IntegrationTestUtils.TEST_START_DATE)
-        run_dict['Survey']['SeasonStartString'] = previous_day_string
+        run_dict['SeasonStartString'] = previous_day_string
 
-        # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
-        os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
-        IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
-        IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)
+        job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+        IntegrationTestUtils.write_json_file(run_dict, job_run_config_file)
+
+        # set the run config file path to the new job-specific file
+        IntegrationTestUtils.RUN_CONFIG_FILE_PATH = job_run_config_file
 
 
     @staticmethod

tests/integration/full/full_test_survey.py

@@ -3,6 +3,7 @@ import os
 import sys
 
 from ews.coordinator.processor_surveys import ProcessorSurveys
+from ews.coordinator.utils.processor_utils import parse_json_file_with_tokens
 from integration.partial.integration_test_utils import IntegrationTestUtils
 from integration.test_suites.survey_test_suite import BaseSurveyTestSuite
 
@@ -28,22 +29,41 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
 
     @staticmethod
     def write_temp_run_config_file():
-        default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
-        default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
-        run_dict: dict = copy.deepcopy(default_config_dict)
-        run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
-        run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
-        run_dict['Survey']['SkipServerDownload'] = False
+
+        """
+        IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH will have been set from the command line args, and point to the
+        production sys config file. We need to modify this file to point to the test output directory.
+        :return:
+        """
+        sys_config = IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH
+        sys_config_dict: dict = parse_json_file_with_tokens(sys_config)
+        sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
+        sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
 
         os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
-        IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
-        IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)
+        job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+        IntegrationTestUtils.write_json_file(sys_config_dict, job_sys_config_file)
+
+        # now set the sys config file path to the new job-specific file
+        IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = job_sys_config_file
+
+        run_config = IntegrationTestUtils.RUN_CONFIG_FILE_PATH
+        run_config_temp_dict: dict = parse_json_file_with_tokens(run_config, sys_config_dict)
+        run_dict: dict = copy.deepcopy(run_config_temp_dict)
+        run_dict['SkipServerDownload'] = False
+
+        job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+        IntegrationTestUtils.write_json_file(run_dict, job_run_config_file)
+
+        # set the run config file path to the new job-specific file
+        IntegrationTestUtils.RUN_CONFIG_FILE_PATH = job_run_config_file
+
 
     @staticmethod
     def run_survey_pipeline():
         survey_processor = ProcessorSurveys()
         IntegrationTestUtils.run_external_pipeline(BaseSurveyTestSuite.SurveyTestSuite.SURVEY_COMPONENT_NAME,
-                                                    BaseSurveyTestSuite.SurveyTestSuite.SURVEY_PROCESSOR_DIR,
+                                                   BaseSurveyTestSuite.SurveyTestSuite.SURVEY_PROCESSOR_DIR,
                                                    IntegrationTestUtils.TEST_START_DATE,
                                                    survey_processor)