From af087dff7c1961be5be0080380d4966c09a95e83 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Fri, 21 Jun 2024 11:14:29 +0100
Subject: [PATCH] more analysis config work
---
.gitignore | 4 +++-
ews/coordinator/processor_base.py | 3 ++-
pipeline_resources/EastAfrica/configs/coordinator/README.txt | 5 +++--
.../{ => analysis}/depo_config_EastAfrica_an_live.json | 4 +---
.../{ => analysis}/env_suit_config_EastAfrica_an_live.json | 2 +-
.../{ => forecast}/depo_config_EastAfrica_fc_live.json | 2 +-
.../{ => forecast}/env_suit_config_EastAfrica_fc_live.json | 2 +-
.../{ => forecast}/epi_config_EastAfrica_fc_live.json | 0
.../met_resample_config_EastAfrica_fc_live.json | 0
.../{ => forecast}/survey_config_EastAfrica_fc_live.json | 0
.../configs/coordinator/template_sys_config_EastAfrica.json | 4 ++--
11 files changed, 14 insertions(+), 12 deletions(-)
rename pipeline_resources/EastAfrica/configs/coordinator/{ => analysis}/depo_config_EastAfrica_an_live.json (86%)
rename pipeline_resources/EastAfrica/configs/coordinator/{ => analysis}/env_suit_config_EastAfrica_an_live.json (97%)
rename pipeline_resources/EastAfrica/configs/coordinator/{ => forecast}/depo_config_EastAfrica_fc_live.json (94%)
rename pipeline_resources/EastAfrica/configs/coordinator/{ => forecast}/env_suit_config_EastAfrica_fc_live.json (99%)
rename pipeline_resources/EastAfrica/configs/coordinator/{ => forecast}/epi_config_EastAfrica_fc_live.json (100%)
rename pipeline_resources/EastAfrica/configs/coordinator/{ => forecast}/met_resample_config_EastAfrica_fc_live.json (100%)
rename pipeline_resources/EastAfrica/configs/coordinator/{ => forecast}/survey_config_EastAfrica_fc_live.json (100%)
diff --git a/.gitignore b/.gitignore
index ead3a8f..86a81b8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,4 +18,6 @@ custom_plotting.py
/docs/_autosummary/
dump.csv
/pipeline_resources/EastAfrica/configs/coordinator/Cred-WRT.json
-/pipeline_resources/EastAfrica/configs/coordinator/sys_config_EastAfrica.json
+/pipeline_resources/EastAfrica/configs/coordinator/forecast/sys_config_EastAfrica_fc_live.json
+/pipeline_resources/EastAfrica/configs/coordinator/forecast/sys_config_EastAfrica_fc_live.json
+/pipeline_resources/EastAfrica/configs/coordinator/analysis/sys_config_EastAfrica_an_live.json
diff --git a/ews/coordinator/processor_base.py b/ews/coordinator/processor_base.py
index 67ee6e0..40105ff 100755
--- a/ews/coordinator/processor_base.py
+++ b/ews/coordinator/processor_base.py
@@ -300,7 +300,8 @@ class ProcessorBase:
if k not in configjson.keys():
configjson[k] = v
else:
- raise Exception(f"Overwriting config key {k} with value from sys_config")
+ logger.warning(f"Key {k} already present in run config - not adding key with same name from "
+ f"the sys_config")
except:
logger.exception(f"Failure in opening or checking config {configtemplate}")
diff --git a/pipeline_resources/EastAfrica/configs/coordinator/README.txt b/pipeline_resources/EastAfrica/configs/coordinator/README.txt
index 7565236..b971764 100644
--- a/pipeline_resources/EastAfrica/configs/coordinator/README.txt
+++ b/pipeline_resources/EastAfrica/configs/coordinator/README.txt
@@ -1,2 +1,3 @@
-make a copy of the template file, You can add a file
-path for your project-level logger or leave blank for a default location
\ No newline at end of file
+make a copy of the template file, You can add a files to set system-specific variables (or sensitive passwords)
+
+the template sys config files will be needed for both forecast and analysis runs,
\ No newline at end of file
diff --git a/pipeline_resources/EastAfrica/configs/coordinator/depo_config_EastAfrica_an_live.json b/pipeline_resources/EastAfrica/configs/coordinator/analysis/depo_config_EastAfrica_an_live.json
similarity index 86%
rename from pipeline_resources/EastAfrica/configs/coordinator/depo_config_EastAfrica_an_live.json
rename to pipeline_resources/EastAfrica/configs/coordinator/analysis/depo_config_EastAfrica_an_live.json
index 4c419ce..c3aaae0 100644
--- a/pipeline_resources/EastAfrica/configs/coordinator/depo_config_EastAfrica_an_live.json
+++ b/pipeline_resources/EastAfrica/configs/coordinator/analysis/depo_config_EastAfrica_an_live.json
@@ -1,6 +1,4 @@
{
- "WorkspacePathout" : "${ProjectRoot}/regions/EastAfrica/workspace_analysis/",
- "WorkspacePath" : "${ProjectRoot}/regions/EastAfrica/workspace_analysis/",
"ServerPathTemplate": "${MetoFTP}/metofficeupload/upload/Ethiopia/fromMO/daily_name/",
"InputFileTemplate" : "WR_NAME_Ethiopia_${StartString}_an",
"DataFileTemplate" : "deposition_srcs_allregions_${StartString}.nc",
@@ -9,7 +7,7 @@
"ProcessInJob" : "process_in_job_dep",
"ProcessEWSPlotting" : "process_EWS_plotting_dep",
"EWS-Plotting" : {
- "SysConfig": "${ConfigsPath}/coordinator/sys_config_EastAfrica.json",
+ "SysConfig": "${ConfigsPath}/coordinator/analysis/sys_config_EastAfrica_an_live.json",
"NameExtractionConfig": "${ConfigsPath}/plotting///name_extraction/NAME_EXTRACTION_CONFIG.json",
"RunConfig": "${ConfigsPath}/plotting///deposition/RUN_CONFIG_DEPO.json",
"RunConfigNorm": "${ConfigsPath}/plotting///deposition/RUN_CONFIG_DEPO_NORMALIZED.json",
diff --git a/pipeline_resources/EastAfrica/configs/coordinator/env_suit_config_EastAfrica_an_live.json b/pipeline_resources/EastAfrica/configs/coordinator/analysis/env_suit_config_EastAfrica_an_live.json
similarity index 97%
rename from pipeline_resources/EastAfrica/configs/coordinator/env_suit_config_EastAfrica_an_live.json
rename to pipeline_resources/EastAfrica/configs/coordinator/analysis/env_suit_config_EastAfrica_an_live.json
index 07072ad..f9c015a 100644
--- a/pipeline_resources/EastAfrica/configs/coordinator/env_suit_config_EastAfrica_an_live.json
+++ b/pipeline_resources/EastAfrica/configs/coordinator/analysis/env_suit_config_EastAfrica_an_live.json
@@ -81,7 +81,7 @@
}
},
"EWS-Plotting" : {
- "SysConfig" : "${ConfigsPath}/coordinator/sys_config_EastAfrica.json",
+ "SysConfig" : "${ConfigsPath}/coordinator/analysis/sys_config_EastAfrica_an_live.json",
"RunConfig" : "${ConfigsPath}/plotting/env_suit/RUN_CONFIG_ENV.json",
"EastAfrica" : {
"ChartConfig" : "${ConfigsPath}/plotting/chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
diff --git a/pipeline_resources/EastAfrica/configs/coordinator/depo_config_EastAfrica_fc_live.json b/pipeline_resources/EastAfrica/configs/coordinator/forecast/depo_config_EastAfrica_fc_live.json
similarity index 94%
rename from pipeline_resources/EastAfrica/configs/coordinator/depo_config_EastAfrica_fc_live.json
rename to pipeline_resources/EastAfrica/configs/coordinator/forecast/depo_config_EastAfrica_fc_live.json
index 36d14e1..ff4ccdb 100644
--- a/pipeline_resources/EastAfrica/configs/coordinator/depo_config_EastAfrica_fc_live.json
+++ b/pipeline_resources/EastAfrica/configs/coordinator/forecast/depo_config_EastAfrica_fc_live.json
@@ -7,7 +7,7 @@
"ProcessInJob": "process_in_job_dep",
"ProcessEWSPlotting": "process_EWS_plotting_dep",
"EWS-Plotting": {
- "SysConfig": "${ConfigsPath}/coordinator/sys_config_EastAfrica.json",
+ "SysConfig": "${ConfigsPath}/coordinator/forecast/sys_config_EastAfrica_fc_live.json",
"NameExtractionConfig": "${ConfigsPath}/plotting///name_extraction/NAME_EXTRACTION_CONFIG.json",
"RunConfig": "${ConfigsPath}/plotting///deposition/RUN_CONFIG_DEPO.json",
"RunConfigNorm": "${ConfigsPath}/plotting///deposition/RUN_CONFIG_DEPO_NORMALIZED.json",
diff --git a/pipeline_resources/EastAfrica/configs/coordinator/env_suit_config_EastAfrica_fc_live.json b/pipeline_resources/EastAfrica/configs/coordinator/forecast/env_suit_config_EastAfrica_fc_live.json
similarity index 99%
rename from pipeline_resources/EastAfrica/configs/coordinator/env_suit_config_EastAfrica_fc_live.json
rename to pipeline_resources/EastAfrica/configs/coordinator/forecast/env_suit_config_EastAfrica_fc_live.json
index 74031e7..2e6205b 100644
--- a/pipeline_resources/EastAfrica/configs/coordinator/env_suit_config_EastAfrica_fc_live.json
+++ b/pipeline_resources/EastAfrica/configs/coordinator/forecast/env_suit_config_EastAfrica_fc_live.json
@@ -79,7 +79,7 @@
}
},
"EWS-Plotting" : {
- "SysConfig" : "${ConfigsPath}/coordinator/sys_config_EastAfrica.json",
+ "SysConfig" : "${ConfigsPath}/coordinator/sys_config_EastAfrica_fc_live.json",
"RunConfig" : "${ConfigsPath}/plotting/env_suit/RUN_CONFIG_ENV.json",
"EastAfrica" : {
"ChartConfig" : "${ConfigsPath}/plotting/chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
diff --git a/pipeline_resources/EastAfrica/configs/coordinator/epi_config_EastAfrica_fc_live.json b/pipeline_resources/EastAfrica/configs/coordinator/forecast/epi_config_EastAfrica_fc_live.json
similarity index 100%
rename from pipeline_resources/EastAfrica/configs/coordinator/epi_config_EastAfrica_fc_live.json
rename to pipeline_resources/EastAfrica/configs/coordinator/forecast/epi_config_EastAfrica_fc_live.json
diff --git a/pipeline_resources/EastAfrica/configs/coordinator/met_resample_config_EastAfrica_fc_live.json b/pipeline_resources/EastAfrica/configs/coordinator/forecast/met_resample_config_EastAfrica_fc_live.json
similarity index 100%
rename from pipeline_resources/EastAfrica/configs/coordinator/met_resample_config_EastAfrica_fc_live.json
rename to pipeline_resources/EastAfrica/configs/coordinator/forecast/met_resample_config_EastAfrica_fc_live.json
diff --git a/pipeline_resources/EastAfrica/configs/coordinator/survey_config_EastAfrica_fc_live.json b/pipeline_resources/EastAfrica/configs/coordinator/forecast/survey_config_EastAfrica_fc_live.json
similarity index 100%
rename from pipeline_resources/EastAfrica/configs/coordinator/survey_config_EastAfrica_fc_live.json
rename to pipeline_resources/EastAfrica/configs/coordinator/forecast/survey_config_EastAfrica_fc_live.json
diff --git a/pipeline_resources/EastAfrica/configs/coordinator/template_sys_config_EastAfrica.json b/pipeline_resources/EastAfrica/configs/coordinator/template_sys_config_EastAfrica.json
index edfec1b..3e9ff6a 100644
--- a/pipeline_resources/EastAfrica/configs/coordinator/template_sys_config_EastAfrica.json
+++ b/pipeline_resources/EastAfrica/configs/coordinator/template_sys_config_EastAfrica.json
@@ -5,8 +5,8 @@
"StartString" : "?",
"ProjectRoot" : "<path to project root>",
"MetoFTP" : "<path to MetoFTP>",
- "WorkspacePathout" : "${ProjectRoot}/regions/EastAfrica/workspace/",
- "WorkspacePath" : "${ProjectRoot}/regions/EastAfrica/workspace/",
+ "WorkspacePathout" : "${ProjectRoot}/regions/EastAfrica/workspace/<analysis or forecast specific sub folder>",
+ "WorkspacePath" : "${ProjectRoot}/regions/EastAfrica/workspace/<analysis or forecast specific sub folder>",
"ConfigsPath" : "${ProjectRoot}/code/coordinator/pipeline_resources/configs",
"AssetsPath" : "${ProjectRoot}/regions/EastAfrica/resources/assets",
"LoggingConfigPath" : "${ProjectRoot}/code/coordinator/pipeline_resources/configs/coordinator/log_config.json",
--
GitLab