From bebf81980b879cd52da3075d61c091667ad3cf91 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Wed, 19 Jun 2024 11:56:07 +0100
Subject: [PATCH] config changes
---
ews/coordinator/processor_environment.py | 7 ++--
.../advisory_config_EastAfrica_fc_live.json | 34 +++++++++----------
.../depo_config_EastAfrica_fc_live.json | 18 +++++-----
.../env_suit_config_EastAfrica_fc_live.json | 19 ++++++-----
.../epi_config_EastAfrica_fc_live.json | 16 ++++-----
.../survey_config_EastAfrica_fc_live.json | 6 ++--
.../template_sys_config_EastAfrica.json | 3 +-
.../configs/plotting/sys/SYS_CONFIG_PINE.json | 2 --
8 files changed, 53 insertions(+), 52 deletions(-)
delete mode 100644 pipeline_resources/configs/plotting/sys/SYS_CONFIG_PINE.json
diff --git a/ews/coordinator/processor_environment.py b/ews/coordinator/processor_environment.py
index 874d271..7fa2cab 100644
--- a/ews/coordinator/processor_environment.py
+++ b/ews/coordinator/processor_environment.py
@@ -167,7 +167,8 @@ class ProcessorEnvironment(ProcessorBase):
# The prevent_overwrite parameter can be set to False if you want to re-run a job in-place.
# Get parameters from the config
- resourcesPath = ProcessorEnvironment.getParameter(pipeline_config, 'RESOURCES_PATH')
+ resourcesPath = ProcessorEnvironment.getParameter(pipeline_config, 'CONFIGS_PATH')
+ assetsPath = ProcessorEnvironment.getParameter(pipeline_config, 'ASSETS_PATH')
workPath = ProcessorEnvironment.getParameter(pipeline_config, 'WORK_PATH') + 'ENVIRONMENT_2.0_' + dateString + '/'
if not os.path.exists(workPath):
os.makedirs(workPath)
@@ -183,7 +184,7 @@ class ProcessorEnvironment(ProcessorBase):
sys_config = ProcessorEnvironment.loadConfig(sys_config_file)
templateName = 'template_' + runType + '_config.json'
- template_configFile = resourcesPath + 'configs/met_extractor/' + templateName
+ template_configFile = resourcesPath + '/met_extractor/' + templateName
config = ProcessorEnvironment.loadConfig(template_configFile)
# Before writing any files, check the output path doesn't exist already
@@ -274,7 +275,7 @@ class ProcessorEnvironment(ProcessorBase):
sense-checking the dimensions of the output
"""
# todo we could swap the spatial points file for a file specifying the expected dimensions - much smaller
- region_spatial_points_file = resourcesPath + 'assets/met_extractor/' + 'input_spatial_points_' + region + '.csv'
+ region_spatial_points_file = assetsPath + '/met_extractor/' + 'input_spatial_points_' + region + '.csv'
spatial_points = pd.read_csv(region_spatial_points_file)
spatial_dim = spatial_points.shape[0]
diff --git a/pipeline_resources/configs/coordinator/advisory_config_EastAfrica_fc_live.json b/pipeline_resources/configs/coordinator/advisory_config_EastAfrica_fc_live.json
index 1030f38..ae89125 100644
--- a/pipeline_resources/configs/coordinator/advisory_config_EastAfrica_fc_live.json
+++ b/pipeline_resources/configs/coordinator/advisory_config_EastAfrica_fc_live.json
@@ -80,44 +80,44 @@
}
},
"Surveys" : {
- "variety_names_tidy_fn" : "${ResourcesPath}/configs/advisory_builder/config_EastAfrica_variety_names_tidy.json",
+ "variety_names_tidy_fn" : "${ConfigsPath}/advisory_builder/config_EastAfrica_variety_names_tidy.json",
"CountryColumnName" : "surveyor_infromation-country",
"EastAfrica": {
"CountryList" : ["Ethiopia", "Kenya", "Tanzania", "Uganda", "Zambia"],
- "ShapeFilenameAdmin1" : "${ResourcesPath}/assets/advisory_builder/gis/eastafrica/eth_adm1_2021_ken_adm1_2019.shp",
- "ShapeFilenameAdmin2" : "${ResourcesPath}/assets/advisory_builder/gis/eastafrica/eth_adm2_2021_ken_adm1_2019_zmb_adm2_2020.shp",
- "ShapeFilenameToAggregate" : "${ResourcesPath}/assets/advisory_builder/gis/eastafrica/eth_adm2_2021_ken_adm1_2019_zmb_adm2_2020.shp",
+ "ShapeFilenameAdmin1" : "${AssetsPath}/advisory_builder/gis/eastafrica/eth_adm1_2021_ken_adm1_2019.shp",
+ "ShapeFilenameAdmin2" : "${AssetsPath}/advisory_builder/gis/eastafrica/eth_adm2_2021_ken_adm1_2019_zmb_adm2_2020.shp",
+ "ShapeFilenameToAggregate" : "${AssetsPath}/advisory_builder/gis/eastafrica/eth_adm2_2021_ken_adm1_2019_zmb_adm2_2020.shp",
"Extent" : [21.5, 47, -18.5, 15],
"LabelZoom" : 5
},
"Ethiopia": {
- "ShapeFilenameAdmin1" : "${ResourcesPath}/assets/advisory_builder/gis/ethiopia/eth_admbnda_adm1_csa_bofedb_2021.shp",
- "ShapeFilenameAdmin2" : "${ResourcesPath}/assets/advisory_builder/gis/ethiopia/eth_admbnda_adm2_csa_bofedb_2021.shp",
- "ShapeFilenameToAggregate" : "${ResourcesPath}/assets/advisory_builder/gis/ethiopia/eth_admbnda_adm2_csa_bofedb_2021.shp",
+ "ShapeFilenameAdmin1" : "${AssetsPath}/advisory_builder/gis/ethiopia/eth_admbnda_adm1_csa_bofedb_2021.shp",
+ "ShapeFilenameAdmin2" : "${AssetsPath}/advisory_builder/gis/ethiopia/eth_admbnda_adm2_csa_bofedb_2021.shp",
+ "ShapeFilenameToAggregate" : "${AssetsPath}/advisory_builder/gis/ethiopia/eth_admbnda_adm2_csa_bofedb_2021.shp",
"Extent" : [32.8, 47, 3.2, 15],
"LabelZoom" : 7
},
"Kenya": {
- "ShapeFilenameAdmin1" : "${ResourcesPath}/assets/advisory_builder/gis/kenya/ken_admbnda_adm1_iebc_20191031_adm2_named.shp",
- "ShapeFilenameAdmin2" : "${ResourcesPath}/assets/advisory_builder/gis/kenya/ken_admbnda_adm2_iebc_20191031.shp",
- "ShapeFilenameToAggregate" : "${ResourcesPath}/assets/advisory_builder/gis/kenya/ken_admbnda_adm1_iebc_20191031_adm2_named.shp",
+ "ShapeFilenameAdmin1" : "${AssetsPath}/advisory_builder/gis/kenya/ken_admbnda_adm1_iebc_20191031_adm2_named.shp",
+ "ShapeFilenameAdmin2" : "${AssetsPath}/advisory_builder/gis/kenya/ken_admbnda_adm2_iebc_20191031.shp",
+ "ShapeFilenameToAggregate" : "${AssetsPath}/advisory_builder/gis/kenya/ken_admbnda_adm1_iebc_20191031_adm2_named.shp",
"Extent" : [33.8, 42, -5, 5.5],
"LabelZoom" : 7
},
"Zambia": {
- "ShapeFilenameAdmin1" : "${ResourcesPath}/assets/advisory_builder/gis/zambia/zmb_admbnda_adm1_dmmu_20201124.shp",
- "ShapeFilenameAdmin2" : "${ResourcesPath}/assets/advisory_builder/gis/zambia/zmb_admbnda_adm2_dmmu_20201124.shp",
- "ShapeFilenameToAggregate" : "${ResourcesPath}/assets/advisory_builder/gis/zambia/zmb_admbnda_adm2_dmmu_20201124.shp",
+ "ShapeFilenameAdmin1" : "${AssetsPath}/advisory_builder/gis/zambia/zmb_admbnda_adm1_dmmu_20201124.shp",
+ "ShapeFilenameAdmin2" : "${AssetsPath}/advisory_builder/gis/zambia/zmb_admbnda_adm2_dmmu_20201124.shp",
+ "ShapeFilenameToAggregate" : "${AssetsPath}/advisory_builder/gis/zambia/zmb_admbnda_adm2_dmmu_20201124.shp",
"Extent" : [21.5, 34, -18.5, -8],
"LabelZoom" : 7
},
"Tanzania": {
- "ShapeFilenameAdmin1" : "${ResourcesPath}/assets/plotting/gis_data/tanzania/tanzania_admin_shape/admin1/tza_admbnda_adm1_20181019.shp",
- "ShapeFilenameAdmin2" : "${ResourcesPath}/assets/plotting/gis_data/tanzania/tanzania_admin_shape/admin2/tza_admbnda_adm2_20181019.shp",
- "ShapeFilenameToAggregate" : "${ResourcesPath}/assets/plotting/gis_data/tanzania/tanzania_admin_shape/admin2/tza_admbnda_adm2_20181019.shp",
+ "ShapeFilenameAdmin1" : "${AssetsPath}/plotting/gis_data/tanzania/tanzania_admin_shape/admin1/tza_admbnda_adm1_20181019.shp",
+ "ShapeFilenameAdmin2" : "${AssetsPath}/plotting/gis_data/tanzania/tanzania_admin_shape/admin2/tza_admbnda_adm2_20181019.shp",
+ "ShapeFilenameToAggregate" : "${AssetsPath}/plotting/gis_data/tanzania/tanzania_admin_shape/admin2/tza_admbnda_adm2_20181019.shp",
"Extent" : [28.93, 41.98, -0.29, -12.37],
"LabelZoom" : 7
}
},
- "logos" : "${ResourcesPath}/assets/advisory_builder/logos_Ethiopia_plotting.png"
+ "logos" : "${AssetsPath}/advisory_builder/logos_Ethiopia_plotting.png"
}
diff --git a/pipeline_resources/configs/coordinator/depo_config_EastAfrica_fc_live.json b/pipeline_resources/configs/coordinator/depo_config_EastAfrica_fc_live.json
index d9222ef..d9cc0b5 100644
--- a/pipeline_resources/configs/coordinator/depo_config_EastAfrica_fc_live.json
+++ b/pipeline_resources/configs/coordinator/depo_config_EastAfrica_fc_live.json
@@ -7,28 +7,28 @@
"ProcessInJob": "process_in_job_dep",
"ProcessEWSPlotting": "process_EWS_plotting_dep",
"EWS-Plotting": {
- "SysConfig": "${ResourcesPath}/configs/coordinator/sys_config_EastAfrica.json",
- "NameExtractionConfig": "${ResourcesPath}/configs/plotting///name_extraction/NAME_EXTRACTION_CONFIG.json",
- "RunConfig": "${ResourcesPath}/configs/plotting///deposition/RUN_CONFIG_DEPO.json",
- "RunConfigNorm": "${ResourcesPath}/configs/plotting///deposition/RUN_CONFIG_DEPO_NORMALIZED.json",
+ "SysConfig": "${ConfigsPath}/coordinator/sys_config_EastAfrica.json",
+ "NameExtractionConfig": "${ConfigsPath}/plotting///name_extraction/NAME_EXTRACTION_CONFIG.json",
+ "RunConfig": "${ConfigsPath}/plotting///deposition/RUN_CONFIG_DEPO.json",
+ "RunConfigNorm": "${ConfigsPath}/plotting///deposition/RUN_CONFIG_DEPO_NORMALIZED.json",
"EastAfrica": {
- "ChartConfig": "${ResourcesPath}/configs/plotting///chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
+ "ChartConfig": "${ConfigsPath}/plotting///chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
"Normalize": "False"
},
"Ethiopia": {
- "ChartConfig": "${ResourcesPath}/configs/plotting///chart/CHART_CONFIG_ETHIOPIA_PINE.json",
+ "ChartConfig": "${ConfigsPath}/plotting///chart/CHART_CONFIG_ETHIOPIA_PINE.json",
"Normalize": "False"
},
"Kenya": {
- "ChartConfig": "${ResourcesPath}/configs/plotting///chart/CHART_CONFIG_KENYA_PINE.json",
+ "ChartConfig": "${ConfigsPath}/plotting///chart/CHART_CONFIG_KENYA_PINE.json",
"Normalize": "False"
},
"Zambia": {
- "ChartConfig": "${ResourcesPath}/configs/plotting///chart/CHART_CONFIG_ZAMBIA_PINE.json",
+ "ChartConfig": "${ConfigsPath}/plotting///chart/CHART_CONFIG_ZAMBIA_PINE.json",
"Normalize": "False"
},
"Tanzania": {
- "ChartConfig": "${ResourcesPath}/configs/plotting///chart/CHART_CONFIG_TANZANIA_EPI.json",
+ "ChartConfig": "${ConfigsPath}/plotting///chart/CHART_CONFIG_TANZANIA_EPI.json",
"Normalize": "False"
}
}
diff --git a/pipeline_resources/configs/coordinator/env_suit_config_EastAfrica_fc_live.json b/pipeline_resources/configs/coordinator/env_suit_config_EastAfrica_fc_live.json
index bd4a23a..74031e7 100644
--- a/pipeline_resources/configs/coordinator/env_suit_config_EastAfrica_fc_live.json
+++ b/pipeline_resources/configs/coordinator/env_suit_config_EastAfrica_fc_live.json
@@ -5,12 +5,13 @@
"ProcessPreJob" : "process_pre_job_server_download",
"ProcessInJob" : "process_in_job_env2_0",
"ProcessEWSPlotting" : "process_EWS_plotting_env2_0",
- "RESOURCES_PATH": "${ResourcesPath}",
+ "CONFIGS_PATH": "${ConfigsPath}",
+ "ASSETS_PATH": "${AssetsPath}",
"WORK_PATH" : "${WorkspacePath}",
"INPUT_PATH" : "${WorkspacePath}",
"OUTPUT_PATH" : "${WorkspacePath}",
- "SYS_CONFIG" : "${ResourcesPath}/configs/met_extractor/SYS_CONFIG_PINE.json",
- "FIELD_NAME_CONSTANTS" : "${ResourcesPath}/configs/met_extractor/FIELD_NAME_CONSTANTS.csv",
+ "SYS_CONFIG" : "${ConfigsPath}/met_extractor/SYS_CONFIG_PINE.json",
+ "FIELD_NAME_CONSTANTS" : "${ConfigsPath}/met_extractor/FIELD_NAME_CONSTANTS.csv",
"RUN_TYPE" : "operational",
"EXTRACTION_DAYS": 7,
"FORECAST_DAYS": 6,
@@ -78,22 +79,22 @@
}
},
"EWS-Plotting" : {
- "SysConfig" : "${ResourcesPath}/configs/coordinator/sys_config_EastAfrica.json",
- "RunConfig" : "${ResourcesPath}/configs/plotting/env_suit/RUN_CONFIG_ENV.json",
+ "SysConfig" : "${ConfigsPath}/coordinator/sys_config_EastAfrica.json",
+ "RunConfig" : "${ConfigsPath}/plotting/env_suit/RUN_CONFIG_ENV.json",
"EastAfrica" : {
- "ChartConfig" : "${ResourcesPath}/configs/plotting/chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
+ "ChartConfig" : "${ConfigsPath}/plotting/chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
"FilterForCountry" : "False"
},
"Ethiopia" : {
- "ChartConfig" : "${ResourcesPath}/configs/plotting/chart/CHART_CONFIG_ETHIOPIA_PINE.json",
+ "ChartConfig" : "${ConfigsPath}/plotting/chart/CHART_CONFIG_ETHIOPIA_PINE.json",
"FilterForCountry" : "True"
},
"Kenya" : {
- "ChartConfig" : "${ResourcesPath}/configs/plotting/chart/CHART_CONFIG_KENYA_PINE.json",
+ "ChartConfig" : "${ConfigsPath}/plotting/chart/CHART_CONFIG_KENYA_PINE.json",
"FilterForCountry" : "True"
},
"Zambia" : {
- "ChartConfig" : "${ResourcesPath}/configs/plotting/chart/CHART_CONFIG_ZAMBIA_PINE.json",
+ "ChartConfig" : "${ConfigsPath}/plotting/chart/CHART_CONFIG_ZAMBIA_PINE.json",
"FilterForCountry" : "True"
}
}
diff --git a/pipeline_resources/configs/coordinator/epi_config_EastAfrica_fc_live.json b/pipeline_resources/configs/coordinator/epi_config_EastAfrica_fc_live.json
index 98b4dbc..b2e88a5 100644
--- a/pipeline_resources/configs/coordinator/epi_config_EastAfrica_fc_live.json
+++ b/pipeline_resources/configs/coordinator/epi_config_EastAfrica_fc_live.json
@@ -8,11 +8,11 @@
"ProcessEWSPlotting" : "process_EWS_plotting_epi",
"Host" : {
"MaxFieldsPerCell" : "1",
- "TargetRaster" : "${ResourcesPath}/assets/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica.tif",
+ "TargetRaster" : "${AssetsPath}/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica.tif",
"HostRasters" : {
- "201001010000" : "${ResourcesPath}/assets/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica.tif",
- "202309200000" : "${ResourcesPath}/assets/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica_20230901.tif",
- "202309260000" : "${ResourcesPath}/assets/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica.tif"
+ "201001010000" : "${AssetsPath}/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica.tif",
+ "202309200000" : "${AssetsPath}/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica_20230901.tif",
+ "202309260000" : "${AssetsPath}/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica.tif"
},
"FileNamePrepared" : "?"
},
@@ -84,10 +84,10 @@
}
],
"EWS-Plotting" : {
- "SysConfig" : "${ResourcesPath}/configs/coordinator/sys_config_EastAfrica.json",
- "RunConfig_seasonsofar" : "${ResourcesPath}/configs/plotting/epi/RUN_CONFIG_EPI.json",
- "RunConfig_seasonplusforecast" : "${ResourcesPath}/configs/plotting/epi/RUN_CONFIG_EPI.json",
- "ChartConfig" : "${ResourcesPath}/configs/plotting/chart/CHART_CONFIG_ETHIOPIA_EPI_PINE.json",
+ "SysConfig" : "${ConfigsPath}/coordinator/sys_config_EastAfrica.json",
+ "RunConfig_seasonsofar" : "${ConfigsPath}/plotting/epi/RUN_CONFIG_EPI.json",
+ "RunConfig_seasonplusforecast" : "${ConfigsPath}/plotting/epi/RUN_CONFIG_EPI.json",
+ "ChartConfig" : "${ConfigsPath}/plotting/chart/CHART_CONFIG_ETHIOPIA_EPI_PINE.json",
"PlottingRegionName" : "Ethiopia",
"EpiCase" : "ps"
}
diff --git a/pipeline_resources/configs/coordinator/survey_config_EastAfrica_fc_live.json b/pipeline_resources/configs/coordinator/survey_config_EastAfrica_fc_live.json
index de0770f..8207088 100644
--- a/pipeline_resources/configs/coordinator/survey_config_EastAfrica_fc_live.json
+++ b/pipeline_resources/configs/coordinator/survey_config_EastAfrica_fc_live.json
@@ -4,7 +4,7 @@
"ProcessEWSPlotting": "process_EWS_plotting_survey",
"AcceptableDowntimeDays": 70,
"SeasonStartString": "20220815",
- "ServerCredentialsFile": "${ResourcesPath}/configs/coordinator/Cred-WRT.json",
+ "ServerCredentialsFile": "${ConfigsPath}/coordinator/Cred-WRT.json",
"Groups": {
"PROD": {
"Origin": [
@@ -72,7 +72,7 @@
"SurveyorNameCol": "surveyor_infromation-surveyor_name",
"SourcesRegionName": "EastAfrica",
"SourcesConfigs": {
- "WRT-Publ": "${ResourcesPath}/configs/source_gen/config_EastAfrica_mapspam2017.json"
+ "WRT-Publ": "${ConfigsPath}/source_gen/config_EastAfrica_mapspam2017.json"
},
- "SourcesConfigDefault": "${ResourcesPath}/configs/source_gen/config_EastAfrica_mapspam2017.json"
+ "SourcesConfigDefault": "${ConfigsPath}/source_gen/config_EastAfrica_mapspam2017.json"
}
\ No newline at end of file
diff --git a/pipeline_resources/configs/coordinator/template_sys_config_EastAfrica.json b/pipeline_resources/configs/coordinator/template_sys_config_EastAfrica.json
index 3efddf3..079a298 100644
--- a/pipeline_resources/configs/coordinator/template_sys_config_EastAfrica.json
+++ b/pipeline_resources/configs/coordinator/template_sys_config_EastAfrica.json
@@ -8,6 +8,7 @@
"LoggingConfigPath" : "${ProjectRoot}/code/coordinator/configs/logger/template_log_config.json",
"WorkspacePathout" : "${ProjectRoot}/regions/EastAfrica/workspace/",
"WorkspacePath" : "${ProjectRoot}/regions/EastAfrica/workspace/",
- "ResourcesPath" : "${ProjectRoot}/regions/EastAfrica/resources/",
+ "ConfigsPath" : "${ProjectRoot}/regions/EastAfrica/resources/configs",
+ "AssetsPath" : "<path to unpacked assets dir>",
"PLACES_SHAPE_FILE": "${ProjectRoot}/regions/EastAfrica/resources/assets/plotting/gis_data/common/places_shape/east_africa_places.shp"
}
diff --git a/pipeline_resources/configs/plotting/sys/SYS_CONFIG_PINE.json b/pipeline_resources/configs/plotting/sys/SYS_CONFIG_PINE.json
deleted file mode 100644
index 7a73a41..0000000
--- a/pipeline_resources/configs/plotting/sys/SYS_CONFIG_PINE.json
+++ /dev/null
@@ -1,2 +0,0 @@
-{
-}
\ No newline at end of file
--
GitLab