From 63b230d208bb9948a483775b1d863e095c0dc31c Mon Sep 17 00:00:00 2001
From: Jake Smith <jws52@cam.ac.uk>
Date: Tue, 4 May 2021 14:00:02 +0100
Subject: [PATCH] feat: Multiple diseases for epi calc
There is a fudge to iterate over diseases, which should be improved.
---
ProcessorComponents.py | 39 +++++++++++++++++++++----
configs/config_Ethiopia_an_ewsdev.json | 2 +-
configs/config_Ethiopia_an_live.json | 2 +-
configs/config_Ethiopia_fc_ewsdev.json | 2 +-
configs/config_Ethiopia_fc_live.json | 2 +-
configs/config_SouthAsia_an_ewsdev.json | 2 +-
configs/config_SouthAsia_an_live.json | 2 +-
configs/config_SouthAsia_big_live.json | 2 +-
configs/config_SouthAsia_fc_ewsdev.json | 2 +-
configs/config_SouthAsia_fc_live.json | 2 +-
10 files changed, 42 insertions(+), 15 deletions(-)
diff --git a/ProcessorComponents.py b/ProcessorComponents.py
index d78dc87..47aa287 100644
--- a/ProcessorComponents.py
+++ b/ProcessorComponents.py
@@ -1111,9 +1111,7 @@ def process_in_job_epi(jobPath,status,config,component):
# initialise any needed variables
- config_epi = config['Epidemiology']
-
- start_date, end_date = calc_epi_date_range(config['StartString'],config_epi['CalculationSpanDays'])
+ start_date, end_date = calc_epi_date_range(config['StartString'],config['Epidemiology']['CalculationSpanDays'])
date_diff = end_date - start_date
@@ -1130,7 +1128,7 @@ def process_in_job_epi(jobPath,status,config,component):
'LeafRust' : 'P_RECONDITA',
'WheatBlast' : 'M_ORYZAE'}
- diseases = config_epi['DiseaseNames']
+ diseases = config['Epidemiology']['DiseaseNames']
# loop over each sub region
@@ -1141,9 +1139,17 @@ def process_in_job_epi(jobPath,status,config,component):
assert disease in disease_latin_name_dict.keys()
+ config_epi = config['Epidemiology'].copy()
+
+ # TODO: CAUTION: Any iterations (e.g. disease or sub-region) are hidden
+ # in jobPath, and not retained in the config file. This is a provlem for
+ # process_EWS_plotting_epi which receives a single config file and must
+ # try a fudge to retrieve details for each iteration.
+ # This should be improved, either by making the one config file
+ # aware of all of the iterations, or looping over iterations in
+ # Processor.py with one iteration-specific config.
case_specific_path = f"{jobPath}/{region}/{disease}/"
Path(case_specific_path).mkdir(parents=True, exist_ok=True)
- # TODO: make config['Epidemiology']['Epi'] formulation(s) disease-specific
logger.info(f"Preparing for epidemiology calc of {disease} in {region}")
@@ -1266,6 +1272,19 @@ def process_in_job_epi(jobPath,status,config,component):
if k not in ['Survey','Environment','Deposition','Epidemiology']:
config_epi[k]=v
+ logger.info('Incremental configuration looks like:')
+ def print_item(item):
+ print(f"Item {item}")
+ print(json.dumps(item,indent=2))
+ def iterate(items):
+ for item in items.items():
+ if hasattr(item,'items'):
+ # iterate
+ iterate(item)
+ else:
+ print_item(item)
+ iterate(config_epi)
+
logger.info('Complete configuration looks like:')
logger.info(json.dumps(config_epi,indent=2))
@@ -1551,7 +1570,7 @@ def process_EWS_plotting_epi(jobPath,config):
# use the first matching epi formulation
# TODO: Is there a more efficient way to select?
epi_filename = [ce['infectionRasterFileName']+'.csv' for ce in config['Epidemiology']['Epi'] if ce['model']==epi_case_operational][0]
-
+
deposition_dir = f"{config['WorkspacePath']}DEPOSITION_{start_string}/WR_NAME_SouthAsia_{start_string}/"
ews_plot_dir = f"{jobPath}/plotting/"
@@ -1563,6 +1582,14 @@ def process_EWS_plotting_epi(jobPath,config):
for disease in diseases:
disease_short = disease.lower().replace('rust','')
+ # a fudge, guess disease type
+ # because config['Epidemiology']['ProcessInJob'] handles disease loop internally
+ # assumes disease name is the last directory before the filename
+ # TODO: handle multiple diseases and regions in Processor as a loop, or in the config
+ disease_to_drop = os.path.dirname(epi_filename).split('/')[-1].replace('Rust','')
+ disease_to_add = disease.replace('Rust','')
+ epi_filename = epi_filename.replace(disease_to_drop,disease_to_add)
+
# prepare command
# TODO: Is output unable to distinguish multiple diseases?
plot_command = [
diff --git a/configs/config_Ethiopia_an_ewsdev.json b/configs/config_Ethiopia_an_ewsdev.json
index fe61f01..632d4ae 100644
--- a/configs/config_Ethiopia_an_ewsdev.json
+++ b/configs/config_Ethiopia_an_ewsdev.json
@@ -97,7 +97,7 @@
}
},
"Epidemiology" : {
- "DiseaseNames" : ["StemRust"],
+ "DiseaseNames" : ["StemRust","StripeRust","LeafRust"],
"CalculationSpanDays" : [0,5],
"TimeStep_hours": "3",
"ProcessPreJob" : "process_pre_job_epi",
diff --git a/configs/config_Ethiopia_an_live.json b/configs/config_Ethiopia_an_live.json
index 95f08b7..c325056 100644
--- a/configs/config_Ethiopia_an_live.json
+++ b/configs/config_Ethiopia_an_live.json
@@ -97,7 +97,7 @@
}
},
"Epidemiology" : {
- "DiseaseNames" : ["StemRust"],
+ "DiseaseNames" : ["StemRust","StripeRust","LeafRust"],
"CalculationSpanDays" : [0,5],
"TimeStep_hours": "3",
"ProcessPreJob" : "process_pre_job_epi",
diff --git a/configs/config_Ethiopia_fc_ewsdev.json b/configs/config_Ethiopia_fc_ewsdev.json
index cf40cb9..121087a 100644
--- a/configs/config_Ethiopia_fc_ewsdev.json
+++ b/configs/config_Ethiopia_fc_ewsdev.json
@@ -97,7 +97,7 @@
}
},
"Epidemiology" : {
- "DiseaseNames" : ["StemRust"],
+ "DiseaseNames" : ["StemRust","StripeRust","LeafRust"],
"CalculationSpanDays" : [0,5],
"TimeStep_hours": "3",
"ProcessPreJob" : "process_pre_job_epi",
diff --git a/configs/config_Ethiopia_fc_live.json b/configs/config_Ethiopia_fc_live.json
index 06a178a..e728fee 100644
--- a/configs/config_Ethiopia_fc_live.json
+++ b/configs/config_Ethiopia_fc_live.json
@@ -97,7 +97,7 @@
}
},
"Epidemiology" : {
- "DiseaseNames" : ["StemRust"],
+ "DiseaseNames" : ["StemRust","StripeRust","LeafRust"],
"CalculationSpanDays" : [0,5],
"TimeStep_hours": "3",
"ProcessPreJob" : "process_pre_job_epi",
diff --git a/configs/config_SouthAsia_an_ewsdev.json b/configs/config_SouthAsia_an_ewsdev.json
index 28753b0..b821d86 100644
--- a/configs/config_SouthAsia_an_ewsdev.json
+++ b/configs/config_SouthAsia_an_ewsdev.json
@@ -106,7 +106,7 @@
}
},
"Epidemiology" : {
- "DiseaseNames" : ["StripeRust"],
+ "DiseaseNames" : ["StripeRust","LeafRust","StemRust"],
"CalculationSpanDays" : [0,5],
"TimeStep_hours": "3",
"ProcessPreJob" : "process_pre_job_epi",
diff --git a/configs/config_SouthAsia_an_live.json b/configs/config_SouthAsia_an_live.json
index 53cccd8..d570960 100644
--- a/configs/config_SouthAsia_an_live.json
+++ b/configs/config_SouthAsia_an_live.json
@@ -106,7 +106,7 @@
}
},
"Epidemiology" : {
- "DiseaseNames" : ["StripeRust"],
+ "DiseaseNames" : ["StripeRust","LeafRust","StemRust"],
"CalculationSpanDays" : [0,5],
"TimeStep_hours": "3",
"ProcessPreJob" : "process_pre_job_epi",
diff --git a/configs/config_SouthAsia_big_live.json b/configs/config_SouthAsia_big_live.json
index 14bdc63..3b42306 100644
--- a/configs/config_SouthAsia_big_live.json
+++ b/configs/config_SouthAsia_big_live.json
@@ -116,7 +116,7 @@
}
},
"Epidemiology" : {
- "DiseaseNames" : ["StripeRust"],
+ "DiseaseNames" : ["StripeRust","LeafRust","StemRust"],
"CalculationSpanDays" : [0,5],
"TimeStep_hours": "3",
"ProcessPreJob" : "process_pre_job_epi",
diff --git a/configs/config_SouthAsia_fc_ewsdev.json b/configs/config_SouthAsia_fc_ewsdev.json
index e7abebe..8c8347c 100644
--- a/configs/config_SouthAsia_fc_ewsdev.json
+++ b/configs/config_SouthAsia_fc_ewsdev.json
@@ -105,7 +105,7 @@
}
},
"Epidemiology" : {
- "DiseaseNames" : ["StripeRust"],
+ "DiseaseNames" : ["StemRust","StripeRust","LeafRust"],
"CalculationSpanDays" : [0,5],
"TimeStep_hours": "3",
"ProcessPreJob" : "process_pre_job_epi",
diff --git a/configs/config_SouthAsia_fc_live.json b/configs/config_SouthAsia_fc_live.json
index 4a4245d..87288ad 100644
--- a/configs/config_SouthAsia_fc_live.json
+++ b/configs/config_SouthAsia_fc_live.json
@@ -106,7 +106,7 @@
}
},
"Epidemiology" : {
- "DiseaseNames" : ["StripeRust"],
+ "DiseaseNames" : ["StripeRust","LeafRust","StemRust"],
"CalculationSpanDays" : [0,5],
"TimeStep_hours": "3",
"ProcessPreJob" : "process_pre_job_epi",
--
GitLab