From 66165c55110aa54f08f719ecf860e7fea413298c Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Thu, 29 Aug 2024 16:19:03 +0100
Subject: [PATCH] making the running of epi plotting dependant on whether
config had been provided. Modifying tests to stop using epi outputs which are
no longer included
---
.gitlab-ci.yml | 2 +-
ews/coordinator/processor_epidemiology.py | 183 +++++++++---------
.../integration/test_suites/epi_test_suite.py | 1 -
3 files changed, 95 insertions(+), 91 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 74ebd7e..c85ccb3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -85,7 +85,7 @@ half_fat_tests:
expire_in: 10 days
half_fat_epi_tests:
- when: manual
+ when: always
stage: half-fat
extends: .before_script
diff --git a/ews/coordinator/processor_epidemiology.py b/ews/coordinator/processor_epidemiology.py
index 15cf9c8..55305f7 100644
--- a/ews/coordinator/processor_epidemiology.py
+++ b/ews/coordinator/processor_epidemiology.py
@@ -616,127 +616,132 @@ class ProcessorEpidemiology(ProcessorBase):
def process_EWS_plotting_epi(self, job_path, config):
- logger.info('started process_EWS_plotting_epi()')
+ """
+ We infer whether to attemp plotting based on the presence or absence of the EWS-Plotting ket in the config.
+ """
+ if 'EWS-Plotting' in config and config['EWS-Plotting'] is not None:
- # initalise necessary variables from config
+ logger.info('started process_EWS_plotting_epi()')
- reference_date_str = config['StartString']
- reference_date = datetime.datetime.strptime(reference_date_str,'%Y%m%d')
-
- start_date, end_date = calc_epi_date_range(reference_date_str,config['CalculationSpanDays'])
+ # initalise necessary variables from config
+ reference_date_str = config['StartString']
+ reference_date = datetime.datetime.strptime(reference_date_str,'%Y%m%d')
- start_string = start_date.strftime('%Y%m%d')
- end_string = end_date.strftime('%Y%m%d')
+ start_date, end_date = calc_epi_date_range(reference_date_str,config['CalculationSpanDays'])
- epi_case_operational = config['EWS-Plotting']['EpiCase']
- if epi_case_operational == 'none':
- logger.info('Config specifies not to call to EWS-Plotting')
- return []
+ start_string = start_date.strftime('%Y%m%d')
+ end_string = end_date.strftime('%Y%m%d')
- diseases = config['DiseaseNames']
+ epi_case_operational = config['EWS-Plotting']['EpiCase']
- # initialise environment
- sys_config = config['EWS-Plotting']['SysConfig']
+ if epi_case_operational == 'none':
+ logger.info('Config specifies not to call to EWS-Plotting')
+ return []
- chart_config = config['EWS-Plotting']['ChartConfig']
+ diseases = config['DiseaseNames']
- # use the first matching epi formulation
- # TODO: Is there a more efficient way to select?
- epi_filename = [ce['infectionRasterFileName'] for ce in config['Epi'] if ce['model']==epi_case_operational][0]
+ # initialise environment
+ sys_config = config['EWS-Plotting']['SysConfig']
- dep_regionnames = ['SouthAsia','Ethiopia']
+ chart_config = config['EWS-Plotting']['ChartConfig']
- # TODO get deposition_dir from config['Deposition']['PathTemplate']
- dep_regionname = 'Ethiopia' #SouthAsia
+ # use the first matching epi formulation
+ # TODO: Is there a more efficient way to select?
+ epi_filename = [ce['infectionRasterFileName'] for ce in config['Epi'] if ce['model']==epi_case_operational][0]
- deposition_dir = f"{config['WorkspacePath']}DEPOSITION_{reference_date_str}/WR_NAME_{dep_regionname}_{reference_date_str}/"
+ dep_regionnames = ['SouthAsia','Ethiopia']
- # TODO: handle multiple diseases and regions in Processor as a loop, or in the config
- deposition_disease_name = [disease_latin_name_dict[disease]+'_DEPOSITION' for disease in diseases][0]
+ # TODO get deposition_dir from config['Deposition']['PathTemplate']
+ dep_regionname = 'Ethiopia' #SouthAsia
- ews_plot_dir = f"{job_path}/plotting/"
+ deposition_dir = f"{config['WorkspacePath']}DEPOSITION_{reference_date_str}/WR_NAME_{dep_regionname}_{reference_date_str}/"
- Path(ews_plot_dir).mkdir(parents=True, exist_ok=True)
+ # TODO: handle multiple diseases and regions in Processor as a loop, or in the config
+ deposition_disease_name = [disease_latin_name_dict[disease]+'_DEPOSITION' for disease in diseases][0]
- # loop over diseases
- ews_plotting_output_globs = []
- for disease in diseases:
- disease_short = disease.lower().replace('rust','')
+ ews_plot_dir = f"{job_path}/plotting/"
- # a fudge, guess disease type
- # because config['ProcessInJob'] handles disease loop internally
- # assumes disease name is the last directory before the filename
- # TODO: handle multiple diseases and regions in Processor as a loop, or in the config
- disease_to_drop = os.path.dirname(epi_filename).split('/')[-1].replace('Rust','')
- disease_to_add = disease.replace('Rust','')
- epi_filename = epi_filename.replace(disease_to_drop,disease_to_add)
+ Path(ews_plot_dir).mkdir(parents=True, exist_ok=True)
- map_title = "Integrated prediction of Wheat $\\bf{" + disease_to_add + "}$ Rust infection"
- if 'PlottingRegionName' not in config['EWS-Plotting']:
- plotting_region_name_lower = config['RegionName'].lower()
- else:
- plotting_region_name_lower = config['EWS-Plotting']['PlottingRegionName'].lower()
+ # loop over diseases
+ ews_plotting_output_globs = []
+ for disease in diseases:
+ disease_short = disease.lower().replace('rust','')
- epi_seasonsofar_fn = epi_filename+'_per_ha_wheat_seasonsofar.csv'
+ # a fudge, guess disease type
+ # because config['ProcessInJob'] handles disease loop internally
+ # assumes disease name is the last directory before the filename
+ # TODO: handle multiple diseases and regions in Processor as a loop, or in the config
+ disease_to_drop = os.path.dirname(epi_filename).split('/')[-1].replace('Rust','')
+ disease_to_add = disease.replace('Rust','')
+ epi_filename = epi_filename.replace(disease_to_drop,disease_to_add)
- epi_seasonincforecast_fn = epi_filename+'_per_ha_wheat.csv'
+ map_title = "Integrated prediction of Wheat $\\bf{" + disease_to_add + "}$ Rust infection"
+ if 'PlottingRegionName' not in config['EWS-Plotting']:
+ plotting_region_name_lower = config['RegionName'].lower()
+ else:
+ plotting_region_name_lower = config['EWS-Plotting']['PlottingRegionName'].lower()
- seasonsofar_run_config = config['EWS-Plotting'].get('RunConfig_seasonsofar',None)
+ epi_seasonsofar_fn = epi_filename+'_per_ha_wheat_seasonsofar.csv'
- # only run season so far (i.e. historic dates) if they exist
- if (seasonsofar_run_config is not None) & os.path.exists(epi_seasonsofar_fn):
+ epi_seasonincforecast_fn = epi_filename+'_per_ha_wheat.csv'
- logger.info(f"Running EWS-Plotting with the following configs:\n{sys_config}\n{seasonsofar_run_config}\n{chart_config}")
+ seasonsofar_run_config = config['EWS-Plotting'].get('RunConfig_seasonsofar',None)
- epi_processor_1 = EPIPostPostProcessor()
- epi_processor_1.set_param_config_files(sys_params_file_arg=sys_config,
+ # only run season so far (i.e. historic dates) if they exist
+ if (seasonsofar_run_config is not None) & os.path.exists(epi_seasonsofar_fn):
+
+ logger.info(f"Running EWS-Plotting with the following configs:\n{sys_config}\n{seasonsofar_run_config}\n{chart_config}")
+
+ epi_processor_1 = EPIPostPostProcessor()
+ epi_processor_1.set_param_config_files(sys_params_file_arg=sys_config,
+ chart_params_file_arg=chart_config,
+ run_params_file_arg=seasonsofar_run_config,
+ epi_input_csv_arg=epi_seasonsofar_fn,
+ disease_type_arg=disease_short+'_seasontodate',
+ issue_date_arg=reference_date_str,
+ output_dir_arg=ews_plot_dir,
+ wheat_sources_dir_arg=deposition_dir,
+ wheat_source_disease_name_arg=deposition_disease_name,
+ map_title_arg=map_title,
+ chart_area_prefix=plotting_region_name_lower)
+ epi_processor_1.process()
+
+ # prepare command for seasonplusforecast
+
+ run_config = config['EWS-Plotting']['RunConfig_seasonplusforecast']
+
+ logger.info(f"Running EWS-Plotting with the following configs:\n{sys_config}\n{run_config}\n{chart_config}")
+
+ epi_processor_2 = EPIPostPostProcessor()
+ epi_processor_2.set_param_config_files(sys_params_file_arg=sys_config,
chart_params_file_arg=chart_config,
- run_params_file_arg=seasonsofar_run_config,
- epi_input_csv_arg=epi_seasonsofar_fn,
- disease_type_arg=disease_short+'_seasontodate',
+ run_params_file_arg=run_config,
+ epi_input_csv_arg=epi_seasonincforecast_fn, # for seasonplusforecast
+ #epi_input_csv_arg=epi_filename+'_weekahead.csv', # for weekahead
+ disease_type_arg=disease_short+'_seasonincforecast',
issue_date_arg=reference_date_str,
output_dir_arg=ews_plot_dir,
wheat_sources_dir_arg=deposition_dir,
wheat_source_disease_name_arg=deposition_disease_name,
map_title_arg=map_title,
chart_area_prefix=plotting_region_name_lower)
- epi_processor_1.process()
-
- # prepare command for seasonplusforecast
-
- run_config = config['EWS-Plotting']['RunConfig_seasonplusforecast']
-
- logger.info(f"Running EWS-Plotting with the following configs:\n{sys_config}\n{run_config}\n{chart_config}")
-
- epi_processor_2 = EPIPostPostProcessor()
- epi_processor_2.set_param_config_files(sys_params_file_arg=sys_config,
- chart_params_file_arg=chart_config,
- run_params_file_arg=run_config,
- epi_input_csv_arg=epi_seasonincforecast_fn, # for seasonplusforecast
- #epi_input_csv_arg=epi_filename+'_weekahead.csv', # for weekahead
- disease_type_arg=disease_short+'_seasonincforecast',
- issue_date_arg=reference_date_str,
- output_dir_arg=ews_plot_dir,
- wheat_sources_dir_arg=deposition_dir,
- wheat_source_disease_name_arg=deposition_disease_name,
- map_title_arg=map_title,
- chart_area_prefix=plotting_region_name_lower)
- epi_processor_2.process()
-
- # check the output
- ews_plotting_output_dir = f"{ews_plot_dir}/images/"
- # TODO: Make this smarter, connected to the results of EWSPlottingEPIBase.plot_epi()
- ews_plotting_output_globs: [str] = []
- ews_plotting_output_globs += [f"{ews_plotting_output_dir}infection_{plotting_region_name_lower}_*{disease_short}*.png"]
-
- ews_plotting_output_globs = get_only_existing_globs(ews_plotting_output_globs,inplace=False)
-
- # check there is some output from EWS-plotting
- if not ews_plotting_output_globs:
- logger.error('EWS-Plotting did not produce any output')
- raise RuntimeError
+ epi_processor_2.process()
+
+ # check the output
+ ews_plotting_output_dir = f"{ews_plot_dir}/images/"
+ # TODO: Make this smarter, connected to the results of EWSPlottingEPIBase.plot_epi()
+ ews_plotting_output_globs: [str] = []
+ ews_plotting_output_globs += [f"{ews_plotting_output_dir}infection_{plotting_region_name_lower}_*{disease_short}*.png"]
+
+ ews_plotting_output_globs = get_only_existing_globs(ews_plotting_output_globs,inplace=False)
+
+ # check there is some output from EWS-plotting
+ if not ews_plotting_output_globs:
+ logger.error('EWS-Plotting did not produce any output')
+ raise RuntimeError
if __name__ == '__main__':
diff --git a/tests/integration/test_suites/epi_test_suite.py b/tests/integration/test_suites/epi_test_suite.py
index ee60bc6..54784cf 100644
--- a/tests/integration/test_suites/epi_test_suite.py
+++ b/tests/integration/test_suites/epi_test_suite.py
@@ -37,7 +37,6 @@ class BaseEpiTestSuite:
"*psbeta*gamma*alpha*")
self.check_wildcard_exists_and_not_empty(alpha_beta_gamma_root, ".csv")
- self.check_wildcard_exists_and_not_empty(alpha_beta_gamma_root, "*comparison.png")
self.check_wildcard_exists_and_not_empty(alpha_beta_gamma_root, "*progression.csv")
self.check_wildcard_exists_and_not_empty(alpha_beta_gamma_root, ".png")
self.check_wildcard_exists_and_not_empty(alpha_beta_gamma_root, ".tif")
--
GitLab