diff --git a/ews/coordinator/processor_base.py b/ews/coordinator/processor_base.py index 5751ca2413fd1827e8f657bc1e58341a62f05f09..344cb3bd2944ad0c1a92c33bab6d93c3ed55a760 100755 --- a/ews/coordinator/processor_base.py +++ b/ews/coordinator/processor_base.py @@ -224,6 +224,11 @@ class ProcessorBase: # load universal configuration sys_config = parse_json_file_with_tokens(sys_config_path) + + # note that file date represents preceding 3 hours, so day's data starts at file timestamp 0300 UTC + start_time: datetime = datetime.datetime.strptime(start_date + '03', '%Y%m%d%H') + start_time_string: str = start_time.strftime('%Y-%m-%d-%H%M') + sys_config['StartTime'] = start_time_string sys_config['StartString'] = start_date # determine job directory @@ -253,10 +258,6 @@ class ProcessorBase: logger.info(f"Job path will be {job_path}") # workspace_path = sys_config['WorkspacePathout'] - # note that file date represents preceding 3 hours, so day's data starts at file timestamp 0300 UTC - start_time: datetime = datetime.datetime.strptime(start_date + '03', '%Y%m%d%H') - start_time_string: str = start_time.strftime('%Y-%m-%d-%H%M') - # run any checks before creating a job directory # if this fails, then make a note once there is a job directory ready = self.process_pre_job(args) @@ -309,9 +310,6 @@ class ProcessorBase: # provide specific case details to template config - configjson['StartTime'] = start_time_string - configjson['StartString'] = start_date - # from configtemplate create configFileName to describe the specific job config_file_name = f"{os.path.basename(configtemplate).replace('.json', '')}_{component}" diff --git a/tests/integration/full/full_test_advisory.py b/tests/integration/full/full_test_advisory.py index 915c146fcfcd956d81f46d91c11a237e72f3bc64..a46ca9029f7473006155f9ff8284822047081219 100644 --- a/tests/integration/full/full_test_advisory.py +++ b/tests/integration/full/full_test_advisory.py @@ -58,8 +58,8 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite): # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True) - IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" - IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" + IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH) @staticmethod diff --git a/tests/integration/full/full_test_deposition.py b/tests/integration/full/full_test_deposition.py index 6c3e0bdf5c2b3563d88dc166d0a969ae87708252..db5529123d5bb3edbaca793bf465f0fb5d5d8427 100644 --- a/tests/integration/full/full_test_deposition.py +++ b/tests/integration/full/full_test_deposition.py @@ -24,6 +24,7 @@ class FullTestDeposition(BaseDepoTestSuite.DepoTestSuite): path = IntegrationTestUtils.TEST_OUT_PATH if path is None \ or not os.listdir(IntegrationTestUtils.TEST_OUT_PATH): + # if True: FullTestDeposition.write_temp_run_config_file() FullTestDeposition.run_depo_pipeline() else: @@ -33,22 +34,33 @@ class FullTestDeposition(BaseDepoTestSuite.DepoTestSuite): @staticmethod def write_temp_run_config_file(): - sys_config = IntegrationTestUtils.DEFAULT_SYS_CONFIG_FILE_PATH + """ + IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH will have been set from the command line args, and point to the + production sys config file. We need to modify this file to point to the test output directory. + :return: + """ + sys_config = IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH sys_config_dict: dict = parse_json_file_with_tokens(sys_config) + sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH + sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH + os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True) - IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" - IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME) + job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" + IntegrationTestUtils.write_json_file(sys_config_dict, job_sys_config_file) + + # now set the sys config file path to the new job-specific file + IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = job_sys_config_file - run_config = IntegrationTestUtils.DEFAULT_DEPO_CONFIG_FILE_PATH - run_config_temp_dict: dict = parse_json_file_with_tokens(run_config) + run_config = IntegrationTestUtils.RUN_CONFIG_FILE_PATH + run_config_temp_dict: dict = parse_json_file_with_tokens(run_config, sys_config_dict) run_dict: dict = copy.deepcopy(run_config_temp_dict) - run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH - run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH + job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" + IntegrationTestUtils.write_json_file(run_dict, job_run_config_file) - IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" - IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME) + # set the run config file path to the new job-specific file + IntegrationTestUtils.RUN_CONFIG_FILE_PATH = job_run_config_file @staticmethod diff --git a/tests/integration/full/full_test_env_suit.py b/tests/integration/full/full_test_env_suit.py index 9537da4565582a23c8f0482aa49c6636bd035ab5..2c3e8295eac541fed1c393973afbf795ce49ab1d 100644 --- a/tests/integration/full/full_test_env_suit.py +++ b/tests/integration/full/full_test_env_suit.py @@ -29,20 +29,26 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite): @staticmethod def write_temp_run_config_file(): - default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH + + sys_config_path = IntegrationTestUtils.DEFAULT_SYS_CONFIG_FILE_PATH + sys_config_dict: dict = IntegrationTestUtils.load_json_file(sys_config_path) + sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH + sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH + IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" + IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH) + + default_config = IntegrationTestUtils.DEFAULT_ENV_SUIT_CONFIG_FILE_PATH default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config) run_dict: dict = copy.deepcopy(default_config_dict) - run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH - run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH - run_dict['Environment']['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH - run_dict['Environment']['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH - run_dict['Environment']['OUTPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH - run_dict['Environment']['EWS-Plotting']['Ethiopia']['FilterForCountry'] = "True" - run_dict['Environment']['EWS-Plotting']['Kenya']['FilterForCountry'] = "True" + run_dict['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH + run_dict['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH + run_dict['OUTPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH + run_dict['EWS-Plotting']['Ethiopia']['FilterForCountry'] = "True" + run_dict['EWS-Plotting']['Kenya']['FilterForCountry'] = "True" os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True) - IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" - IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" + IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH) @staticmethod diff --git a/tests/integration/full/full_test_epi.py b/tests/integration/full/full_test_epi.py index 9acfd8edd25423c1ca2003294c680d680b9e11d3..7a4063c30e272ca5d56f72c24542b401c9a0a2d5 100644 --- a/tests/integration/full/full_test_epi.py +++ b/tests/integration/full/full_test_epi.py @@ -64,8 +64,8 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite): # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True) - IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" - IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" + IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH) @staticmethod diff --git a/tests/integration/full/full_test_survey.py b/tests/integration/full/full_test_survey.py index 9cd082ef5d3f8b6be74d23f6e6e7e40a26f82e54..bd2b23050a747f4797e2d1702a180d009c6a73aa 100644 --- a/tests/integration/full/full_test_survey.py +++ b/tests/integration/full/full_test_survey.py @@ -36,8 +36,8 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite): run_dict['Survey']['SkipServerDownload'] = False os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True) - IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" - IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" + IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH) @staticmethod def run_survey_pipeline(): diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py index ee8c64e7992a1e76fd808c857d3d8d833b17b027..ebc59a0613c19ab13f463b8360a44e77328b4377 100644 --- a/tests/integration/partial/integration_test_utils.py +++ b/tests/integration/partial/integration_test_utils.py @@ -25,8 +25,13 @@ class IntegrationTestUtils: DEFAULT_ADVISORY_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/advisory_config_EastAfrica_fc_live.json" TEST_WORKSPACE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/workspace/" - TEMP_SYS_CONFIG_FILE_NAME: str = None - TEMP_RUN_CONFIG_FILE_NAME: str = None + + """ + The RUN_SYS_CONFIG_FILE_PATH and RUN_CONFIG_FILE_PATH variables are used to store the file paths of the json configs + that are written in the job dir for a specific test run. They are used by partial and full integration tests. + """ + RUN_SYS_CONFIG_FILE_PATH: str = None + RUN_CONFIG_FILE_PATH: str = None TEST_ASSETS_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/assets/coordinator/" EXAMPLE_SURVEY_FILE_PATH: str = TEST_ASSETS_PATH + "example_survey_run.zip" @@ -52,6 +57,7 @@ class IntegrationTestUtils: @staticmethod def build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() + parser.add_argument('--sys_config', required = True) parser.add_argument('--config', required = True) parser.add_argument('--outdir', required = True) parser.add_argument('--email_cred', required = True) @@ -80,14 +86,21 @@ class IntegrationTestUtils: _parser = IntegrationTestUtils.build_arg_parser() _args = _parser.parse_args() - _config_file: str = _args.config + _sys_config_file: str = _args.sys_config + _run_config_file: str = _args.config _outdir: str = _args.outdir _email_cred_path: str = _args.email_cred _test_report_dir: str = _args.test_report_dir _run_date_type: str = _args.run_date_type _custom_run_date: str = _args.custom_run_date - IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH = _config_file + """ + We store paths to the production run json files here, these are then read into a dict, output paths are + overridden to point to the test dir, then re-written as the json config in the test dir. When the test-specific + files are written, the paths are overridden to point to the test dir files. + """ + IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = _sys_config_file + IntegrationTestUtils.RUN_CONFIG_FILE_PATH = _run_config_file nowstring: str = IntegrationTestUtils.get_now_string() prefix: str = f"temp_{test_prefix}_" + nowstring @@ -197,8 +210,8 @@ class IntegrationTestUtils: args_dict: dict = {} - sys_config_path = IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME - config_paths = [IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME] + sys_config_path = IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH + config_paths = [IntegrationTestUtils.RUN_CONFIG_FILE_PATH] # note, possible to override these values in the kwargs loop below args_dict['live'] = False @@ -216,12 +229,11 @@ class IntegrationTestUtils: # need EMAIL_CRED in the environment before we run a Processor os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH - # need LOGGING_CONFIG_PATH in the environment before we run a Processor - os.environ["LOGGING_CONFIG"] = IntegrationTestUtils.LOGGING_CONFIG_PATH try: processor.run_process(args_dict) - except SystemExit: + except SystemExit as e: + print(f"SystemExit: {e}") # we will eventually want to throw these to the calling class to be dealt with pass @@ -240,7 +252,8 @@ class IntegrationTestUtils: args_dict['start_date'] = start_date args_dict['component'] = component args_dict['short_name'] = short_name - args_dict['config_paths'] = [IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME] + args_dict['sys_config_path'] = IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH + args_dict['config_paths'] = [IntegrationTestUtils.RUN_CONFIG_FILE_PATH] args_dict['log_level'] = 'info' args_dict['clearup'] = True @@ -249,8 +262,6 @@ class IntegrationTestUtils: # need EMAIL_CRED in the environment before we run a Processor os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH - # need LOGGING_CONFIG_PATH in the environment before we run a Processor - os.environ["LOGGING_CONFIG"] = IntegrationTestUtils.LOGGING_CONFIG_PATH try: processor.run_process(args_dict) diff --git a/tests/integration/partial/test_advisory.py b/tests/integration/partial/test_advisory.py index 79c86e0716dbdd27ff07f9d2d916442a8a74f349..5ecbd3311b352e1ea8b69a0bbdc1566ec31081fa 100644 --- a/tests/integration/partial/test_advisory.py +++ b/tests/integration/partial/test_advisory.py @@ -51,8 +51,8 @@ class TestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite): sys_config_dict: dict = IntegrationTestUtils.load_json_file(sys_config_path) sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH - IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" - IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" + IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH) run_config = IntegrationTestUtils.DEFAULT_ADVISORY_CONFIG_FILE_PATH run_config_dict: dict = IntegrationTestUtils.load_json_file(run_config) @@ -60,8 +60,8 @@ class TestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite): # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True) - IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" - IntegrationTestUtils.write_json_file(run_config_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" + IntegrationTestUtils.write_json_file(run_config_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH) @staticmethod diff --git a/tests/integration/partial/test_deposition.py b/tests/integration/partial/test_deposition.py index ac21193d3b3230783dd391377495eb7dc32ae497..23bc795b196afcee7d1ce53f032823cfee0a25dc 100644 --- a/tests/integration/partial/test_deposition.py +++ b/tests/integration/partial/test_deposition.py @@ -53,8 +53,8 @@ class TestDeposition(BaseDepoTestSuite.DepoTestSuite): sys_config_dict: dict = IntegrationTestUtils.load_json_file(sys_config_path) sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH - IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" - IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" + IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH) run_config_path = IntegrationTestUtils.DEFAULT_DEPO_CONFIG_FILE_PATH run_config_dict: dict = IntegrationTestUtils.load_json_file(run_config_path) @@ -62,8 +62,8 @@ class TestDeposition(BaseDepoTestSuite.DepoTestSuite): full_server_path = os.path.abspath(IntegrationTestUtils.TEST_ASSETS_PATH) run_config_dict['ServerPathTemplate'] = full_server_path - IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" - IntegrationTestUtils.write_json_file(run_config_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" + IntegrationTestUtils.write_json_file(run_config_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH) @staticmethod diff --git a/tests/integration/partial/test_env_suit.py b/tests/integration/partial/test_env_suit.py index 9153a62582957213100b96def13b1ec66609c94c..34707763b57023686b72aa70ce0b4a8e4552f98f 100644 --- a/tests/integration/partial/test_env_suit.py +++ b/tests/integration/partial/test_env_suit.py @@ -53,8 +53,8 @@ class TestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite): sys_config_dict: dict = IntegrationTestUtils.load_json_file(sys_config_path) sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH - IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" - IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" + IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH) run_config_path = IntegrationTestUtils.DEFAULT_ENV_SUIT_CONFIG_FILE_PATH run_config_dict: dict = IntegrationTestUtils.load_json_file(run_config_path) @@ -65,8 +65,8 @@ class TestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite): full_server_path = os.path.abspath(IntegrationTestUtils.TEST_ASSETS_PATH) run_config_dict['ServerPathTemplate'] = full_server_path - IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" - IntegrationTestUtils.write_json_file(run_config_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" + IntegrationTestUtils.write_json_file(run_config_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH) @staticmethod diff --git a/tests/integration/partial/test_epi.py b/tests/integration/partial/test_epi.py index 8a61e8da178c0302b3cb3ce8b5dc1980bc968992..afb85d8b371134c84dbfd08d58d3111a8f14fac0 100644 --- a/tests/integration/partial/test_epi.py +++ b/tests/integration/partial/test_epi.py @@ -48,16 +48,16 @@ class TestEpi(BaseEpiTestSuite.EpiTestSuite): sys_config_dict: dict = IntegrationTestUtils.load_json_file(sys_config_path) sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH - IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" - IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" + IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH) run_config = IntegrationTestUtils.DEFAULT_EPI_CONFIG_FILE_PATH run_config_dict: dict = IntegrationTestUtils.load_json_file(run_config) run_config_dict['ServerName'] = '' # nothing, as local machine run_config_dict['CalculationSpanDays'] = [0, 1] - IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" - IntegrationTestUtils.write_json_file(run_config_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" + IntegrationTestUtils.write_json_file(run_config_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH) @staticmethod diff --git a/tests/integration/partial/test_survey.py b/tests/integration/partial/test_survey.py index 59edcf4e4812ee6c64e470dc17440565e7fbda9e..3946467a7ed7ae4be8222ecccadc17acb818885c 100644 --- a/tests/integration/partial/test_survey.py +++ b/tests/integration/partial/test_survey.py @@ -49,8 +49,8 @@ class TestSurvey(BaseSurveyTestSuite.SurveyTestSuite): sys_config_dict: dict = IntegrationTestUtils.load_json_file(sys_config_path) sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH - IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" - IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" + IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH) run_config = IntegrationTestUtils.DEFAULT_SURVEY_CONFIG_FILE_PATH run_config_dict: dict = IntegrationTestUtils.load_json_file(run_config) @@ -63,8 +63,8 @@ class TestSurvey(BaseSurveyTestSuite.SurveyTestSuite): # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True) - IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" - IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME) + IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" + IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH) @staticmethod def unpack_dependencies(): diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/delete_config_EastAfrica_fc_live.json similarity index 100% rename from tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/config_EastAfrica_fc_live.json rename to tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/delete_config_EastAfrica_fc_live.json diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/sys_config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/sys_config_EastAfrica_fc_live.json index 5a979ca832dc1f4644b35618f729b5ec151d8cb9..84af231080046362df257b86dc5b1a9e2d9078ba 100644 --- a/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/sys_config_EastAfrica_fc_live.json +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/sys_config_EastAfrica_fc_live.json @@ -1,8 +1,6 @@ { "RegionName" : "EastAfrica", "SubRegionNames" : ["EastAfrica","Ethiopia"], - "StartTime" : "?", - "StartString" : "?", "ProjectRoot" : "../../test_data/test_deployment/", "MetoFTP" : "<path to MetoFTP>", "WorkspacePathout" : "set_in_the_code", diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/source_gen/config_EastAfrica_mapspam2017.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/source_gen/config_EastAfrica_mapspam2017.json index 9f3135b7b7c73a5e9a2c4c7b3f3cce9e2110223d..3f634b21b9c56f01f956911988dceaf9070ec8df 100644 --- a/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/source_gen/config_EastAfrica_mapspam2017.json +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/source_gen/config_EastAfrica_mapspam2017.json @@ -3,11 +3,11 @@ "wheat_source_name" : "MAPSPAM2017", "wheat_file" : "NA", "wheat_rasters_bydate" : { - "20170101" : "${ProjectRoot}/regions/EastAfrica/resources/assets/source_gen/spam2017V1r1_SSA_gr_H_WHEA_A_clipEastAfrica.tif" + "20170101" : "${ProjectRoot}/regions/EastAfrica/resources/assets/source_gen/spam2017V1r1_SSA_gr_H_WHEA_A_small.tif" }, - "cluster_poly_file" : "${ProjectRoot}/regions/EastAfrica/resources/assets/source_gen/boundaries/UMG_Mk11_grid_EastAfrica_withregions.shp", + "cluster_poly_file" : "${ProjectRoot}/regions/EastAfrica/resources/assets/source_gen/boundaries/EastAfrica_clusters_small.shp", "id_col" : "id", - "region_file" : "${ProjectRoot}/regions/EastAfrica/resources/assets/source_gen/UMG_Mk11_grid_EastAfrica_withregions.shp", + "region_file" : "${ProjectRoot}/regions/EastAfrica/resources/assets/source_gen/boundaries/EastAfrica_regions_small.shp", "region_id_col" : "REGION_ID", "wheat_stages_file" : "${ProjectRoot}/regions/EastAfrica/resources/assets/source_gen/approx_growth_timing_cleaned.csv" }