config changes

ews/coordinator/processor_environment.py

@@ -167,7 +167,8 @@ class ProcessorEnvironment(ProcessorBase):
         # The prevent_overwrite parameter can be set to False if you want to re-run a job in-place.
 
         # Get parameters from the config
-        resourcesPath = ProcessorEnvironment.getParameter(pipeline_config, 'RESOURCES_PATH')
+        resourcesPath = ProcessorEnvironment.getParameter(pipeline_config, 'CONFIGS_PATH')
+        assetsPath = ProcessorEnvironment.getParameter(pipeline_config, 'ASSETS_PATH')
         workPath = ProcessorEnvironment.getParameter(pipeline_config, 'WORK_PATH') + 'ENVIRONMENT_2.0_' + dateString + '/'
         if not os.path.exists(workPath):
             os.makedirs(workPath)
@@ -183,7 +184,7 @@ class ProcessorEnvironment(ProcessorBase):
         sys_config = ProcessorEnvironment.loadConfig(sys_config_file)
 
         templateName = 'template_' + runType + '_config.json'
-        template_configFile = resourcesPath + 'configs/met_extractor/' + templateName
+        template_configFile = resourcesPath + '/met_extractor/' + templateName
         config = ProcessorEnvironment.loadConfig(template_configFile)
 
         # Before writing any files, check the output path doesn't exist already
@@ -274,7 +275,7 @@ class ProcessorEnvironment(ProcessorBase):
                     sense-checking the dimensions of the output
                     """
                     # todo we could swap the spatial points file for a file specifying the expected dimensions - much smaller
-                    region_spatial_points_file = resourcesPath + 'assets/met_extractor/' + 'input_spatial_points_' + region + '.csv'
+                    region_spatial_points_file = assetsPath + '/met_extractor/' + 'input_spatial_points_' + region + '.csv'
                     spatial_points = pd.read_csv(region_spatial_points_file)
                     spatial_dim = spatial_points.shape[0]
 

pipeline_resources/configs/coordinator/advisory_config_EastAfrica_fc_live.json

@@ -80,44 +80,44 @@
         }
     },
     "Surveys" : {
-            "variety_names_tidy_fn" : "${ResourcesPath}/configs/advisory_builder/config_EastAfrica_variety_names_tidy.json",
+            "variety_names_tidy_fn" : "${ConfigsPath}/advisory_builder/config_EastAfrica_variety_names_tidy.json",
             "CountryColumnName" : "surveyor_infromation-country",
             "EastAfrica": {
                 "CountryList" : ["Ethiopia", "Kenya", "Tanzania", "Uganda", "Zambia"],
-                "ShapeFilenameAdmin1" : "${ResourcesPath}/assets/advisory_builder/gis/eastafrica/eth_adm1_2021_ken_adm1_2019.shp",
-                "ShapeFilenameAdmin2" : "${ResourcesPath}/assets/advisory_builder/gis/eastafrica/eth_adm2_2021_ken_adm1_2019_zmb_adm2_2020.shp",
-                "ShapeFilenameToAggregate" : "${ResourcesPath}/assets/advisory_builder/gis/eastafrica/eth_adm2_2021_ken_adm1_2019_zmb_adm2_2020.shp",
+                "ShapeFilenameAdmin1" : "${AssetsPath}/advisory_builder/gis/eastafrica/eth_adm1_2021_ken_adm1_2019.shp",
+                "ShapeFilenameAdmin2" : "${AssetsPath}/advisory_builder/gis/eastafrica/eth_adm2_2021_ken_adm1_2019_zmb_adm2_2020.shp",
+                "ShapeFilenameToAggregate" : "${AssetsPath}/advisory_builder/gis/eastafrica/eth_adm2_2021_ken_adm1_2019_zmb_adm2_2020.shp",
                 "Extent" : [21.5, 47, -18.5, 15],
                 "LabelZoom" : 5
             },
             "Ethiopia": {
-                "ShapeFilenameAdmin1" : "${ResourcesPath}/assets/advisory_builder/gis/ethiopia/eth_admbnda_adm1_csa_bofedb_2021.shp",
-                "ShapeFilenameAdmin2" : "${ResourcesPath}/assets/advisory_builder/gis/ethiopia/eth_admbnda_adm2_csa_bofedb_2021.shp",
-                "ShapeFilenameToAggregate" : "${ResourcesPath}/assets/advisory_builder/gis/ethiopia/eth_admbnda_adm2_csa_bofedb_2021.shp",
+                "ShapeFilenameAdmin1" : "${AssetsPath}/advisory_builder/gis/ethiopia/eth_admbnda_adm1_csa_bofedb_2021.shp",
+                "ShapeFilenameAdmin2" : "${AssetsPath}/advisory_builder/gis/ethiopia/eth_admbnda_adm2_csa_bofedb_2021.shp",
+                "ShapeFilenameToAggregate" : "${AssetsPath}/advisory_builder/gis/ethiopia/eth_admbnda_adm2_csa_bofedb_2021.shp",
                 "Extent" : [32.8, 47, 3.2, 15],
                 "LabelZoom" : 7
             },
             "Kenya": {
-                "ShapeFilenameAdmin1" : "${ResourcesPath}/assets/advisory_builder/gis/kenya/ken_admbnda_adm1_iebc_20191031_adm2_named.shp",
-                "ShapeFilenameAdmin2" : "${ResourcesPath}/assets/advisory_builder/gis/kenya/ken_admbnda_adm2_iebc_20191031.shp",
-                "ShapeFilenameToAggregate" : "${ResourcesPath}/assets/advisory_builder/gis/kenya/ken_admbnda_adm1_iebc_20191031_adm2_named.shp",
+                "ShapeFilenameAdmin1" : "${AssetsPath}/advisory_builder/gis/kenya/ken_admbnda_adm1_iebc_20191031_adm2_named.shp",
+                "ShapeFilenameAdmin2" : "${AssetsPath}/advisory_builder/gis/kenya/ken_admbnda_adm2_iebc_20191031.shp",
+                "ShapeFilenameToAggregate" : "${AssetsPath}/advisory_builder/gis/kenya/ken_admbnda_adm1_iebc_20191031_adm2_named.shp",
                 "Extent" : [33.8, 42, -5, 5.5],
                 "LabelZoom" : 7
             },
             "Zambia": {
-                "ShapeFilenameAdmin1" : "${ResourcesPath}/assets/advisory_builder/gis/zambia/zmb_admbnda_adm1_dmmu_20201124.shp",
-                "ShapeFilenameAdmin2" : "${ResourcesPath}/assets/advisory_builder/gis/zambia/zmb_admbnda_adm2_dmmu_20201124.shp",
-                "ShapeFilenameToAggregate" : "${ResourcesPath}/assets/advisory_builder/gis/zambia/zmb_admbnda_adm2_dmmu_20201124.shp",
+                "ShapeFilenameAdmin1" : "${AssetsPath}/advisory_builder/gis/zambia/zmb_admbnda_adm1_dmmu_20201124.shp",
+                "ShapeFilenameAdmin2" : "${AssetsPath}/advisory_builder/gis/zambia/zmb_admbnda_adm2_dmmu_20201124.shp",
+                "ShapeFilenameToAggregate" : "${AssetsPath}/advisory_builder/gis/zambia/zmb_admbnda_adm2_dmmu_20201124.shp",
                 "Extent" : [21.5, 34, -18.5, -8],
                 "LabelZoom" : 7
             },
             "Tanzania": {
-                "ShapeFilenameAdmin1" : "${ResourcesPath}/assets/plotting/gis_data/tanzania/tanzania_admin_shape/admin1/tza_admbnda_adm1_20181019.shp",
-                "ShapeFilenameAdmin2" : "${ResourcesPath}/assets/plotting/gis_data/tanzania/tanzania_admin_shape/admin2/tza_admbnda_adm2_20181019.shp",
-                "ShapeFilenameToAggregate" : "${ResourcesPath}/assets/plotting/gis_data/tanzania/tanzania_admin_shape/admin2/tza_admbnda_adm2_20181019.shp",
+                "ShapeFilenameAdmin1" : "${AssetsPath}/plotting/gis_data/tanzania/tanzania_admin_shape/admin1/tza_admbnda_adm1_20181019.shp",
+                "ShapeFilenameAdmin2" : "${AssetsPath}/plotting/gis_data/tanzania/tanzania_admin_shape/admin2/tza_admbnda_adm2_20181019.shp",
+                "ShapeFilenameToAggregate" : "${AssetsPath}/plotting/gis_data/tanzania/tanzania_admin_shape/admin2/tza_admbnda_adm2_20181019.shp",
                 "Extent" : [28.93, 41.98, -0.29, -12.37],
                 "LabelZoom" : 7
             }
     },
-    "logos" : "${ResourcesPath}/assets/advisory_builder/logos_Ethiopia_plotting.png"
+    "logos" : "${AssetsPath}/advisory_builder/logos_Ethiopia_plotting.png"
 }

pipeline_resources/configs/coordinator/depo_config_EastAfrica_fc_live.json

@@ -7,28 +7,28 @@
     "ProcessInJob": "process_in_job_dep",
     "ProcessEWSPlotting": "process_EWS_plotting_dep",
     "EWS-Plotting": {
-        "SysConfig": "${ResourcesPath}/configs/coordinator/sys_config_EastAfrica.json",
-        "NameExtractionConfig": "${ResourcesPath}/configs/plotting///name_extraction/NAME_EXTRACTION_CONFIG.json",
-        "RunConfig": "${ResourcesPath}/configs/plotting///deposition/RUN_CONFIG_DEPO.json",
-        "RunConfigNorm": "${ResourcesPath}/configs/plotting///deposition/RUN_CONFIG_DEPO_NORMALIZED.json",
+        "SysConfig": "${ConfigsPath}/coordinator/sys_config_EastAfrica.json",
+        "NameExtractionConfig": "${ConfigsPath}/plotting///name_extraction/NAME_EXTRACTION_CONFIG.json",
+        "RunConfig": "${ConfigsPath}/plotting///deposition/RUN_CONFIG_DEPO.json",
+        "RunConfigNorm": "${ConfigsPath}/plotting///deposition/RUN_CONFIG_DEPO_NORMALIZED.json",
         "EastAfrica": {
-            "ChartConfig": "${ResourcesPath}/configs/plotting///chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
+            "ChartConfig": "${ConfigsPath}/plotting///chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
             "Normalize": "False"
         },
         "Ethiopia": {
-            "ChartConfig": "${ResourcesPath}/configs/plotting///chart/CHART_CONFIG_ETHIOPIA_PINE.json",
+            "ChartConfig": "${ConfigsPath}/plotting///chart/CHART_CONFIG_ETHIOPIA_PINE.json",
             "Normalize": "False"
         },
         "Kenya": {
-            "ChartConfig": "${ResourcesPath}/configs/plotting///chart/CHART_CONFIG_KENYA_PINE.json",
+            "ChartConfig": "${ConfigsPath}/plotting///chart/CHART_CONFIG_KENYA_PINE.json",
             "Normalize": "False"
         },
         "Zambia": {
-            "ChartConfig": "${ResourcesPath}/configs/plotting///chart/CHART_CONFIG_ZAMBIA_PINE.json",
+            "ChartConfig": "${ConfigsPath}/plotting///chart/CHART_CONFIG_ZAMBIA_PINE.json",
             "Normalize": "False"
         },
         "Tanzania": {
-            "ChartConfig": "${ResourcesPath}/configs/plotting///chart/CHART_CONFIG_TANZANIA_EPI.json",
+            "ChartConfig": "${ConfigsPath}/plotting///chart/CHART_CONFIG_TANZANIA_EPI.json",
             "Normalize": "False"
         }
     }

pipeline_resources/configs/coordinator/env_suit_config_EastAfrica_fc_live.json

@@ -5,12 +5,13 @@
     "ProcessPreJob" : "process_pre_job_server_download",
     "ProcessInJob" : "process_in_job_env2_0",
     "ProcessEWSPlotting" : "process_EWS_plotting_env2_0",
-    "RESOURCES_PATH": "${ResourcesPath}",
+    "CONFIGS_PATH": "${ConfigsPath}",
+    "ASSETS_PATH": "${AssetsPath}",
     "WORK_PATH" : "${WorkspacePath}",
     "INPUT_PATH" : "${WorkspacePath}",
     "OUTPUT_PATH" : "${WorkspacePath}",
-    "SYS_CONFIG" : "${ResourcesPath}/configs/met_extractor/SYS_CONFIG_PINE.json",
-    "FIELD_NAME_CONSTANTS" : "${ResourcesPath}/configs/met_extractor/FIELD_NAME_CONSTANTS.csv",
+    "SYS_CONFIG" : "${ConfigsPath}/met_extractor/SYS_CONFIG_PINE.json",
+    "FIELD_NAME_CONSTANTS" : "${ConfigsPath}/met_extractor/FIELD_NAME_CONSTANTS.csv",
     "RUN_TYPE" : "operational",
     "EXTRACTION_DAYS": 7,
     "FORECAST_DAYS": 6,
@@ -78,22 +79,22 @@
         }
     },
     "EWS-Plotting" : {
-        "SysConfig" : "${ResourcesPath}/configs/coordinator/sys_config_EastAfrica.json",
-        "RunConfig" : "${ResourcesPath}/configs/plotting/env_suit/RUN_CONFIG_ENV.json",
+        "SysConfig" : "${ConfigsPath}/coordinator/sys_config_EastAfrica.json",
+        "RunConfig" : "${ConfigsPath}/plotting/env_suit/RUN_CONFIG_ENV.json",
         "EastAfrica" : {
-            "ChartConfig" : "${ResourcesPath}/configs/plotting/chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
+            "ChartConfig" : "${ConfigsPath}/plotting/chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
             "FilterForCountry" : "False"
         },
         "Ethiopia" : {
-            "ChartConfig" : "${ResourcesPath}/configs/plotting/chart/CHART_CONFIG_ETHIOPIA_PINE.json",
+            "ChartConfig" : "${ConfigsPath}/plotting/chart/CHART_CONFIG_ETHIOPIA_PINE.json",
             "FilterForCountry" : "True"
         },
         "Kenya" : {
-            "ChartConfig" : "${ResourcesPath}/configs/plotting/chart/CHART_CONFIG_KENYA_PINE.json",
+            "ChartConfig" : "${ConfigsPath}/plotting/chart/CHART_CONFIG_KENYA_PINE.json",
             "FilterForCountry" : "True"
         },
         "Zambia" : {
-            "ChartConfig" : "${ResourcesPath}/configs/plotting/chart/CHART_CONFIG_ZAMBIA_PINE.json",
+            "ChartConfig" : "${ConfigsPath}/plotting/chart/CHART_CONFIG_ZAMBIA_PINE.json",
             "FilterForCountry" : "True"
         }
     }

pipeline_resources/configs/coordinator/epi_config_EastAfrica_fc_live.json

@@ -8,11 +8,11 @@
     "ProcessEWSPlotting" : "process_EWS_plotting_epi",
     "Host" : {
         "MaxFieldsPerCell" : "1",
-        "TargetRaster" : "${ResourcesPath}/assets/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica.tif",
+        "TargetRaster" : "${AssetsPath}/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica.tif",
     "HostRasters" : {
-            "201001010000" : "${ResourcesPath}/assets/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica.tif",
-            "202309200000" : "${ResourcesPath}/assets/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica_20230901.tif",
-            "202309260000" : "${ResourcesPath}/assets/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica.tif"
+            "201001010000" : "${AssetsPath}/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica.tif",
+            "202309200000" : "${AssetsPath}/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica_20230901.tif",
+            "202309260000" : "${AssetsPath}/epimodel/wheat_area_frac_MapSPAM2010_EastAfrica.tif"
         },
     "FileNamePrepared" : "?"
     },
@@ -84,10 +84,10 @@
         }
     ],
     "EWS-Plotting" : {
-        "SysConfig" : "${ResourcesPath}/configs/coordinator/sys_config_EastAfrica.json",
-        "RunConfig_seasonsofar" : "${ResourcesPath}/configs/plotting/epi/RUN_CONFIG_EPI.json",
-        "RunConfig_seasonplusforecast" : "${ResourcesPath}/configs/plotting/epi/RUN_CONFIG_EPI.json",
-        "ChartConfig" : "${ResourcesPath}/configs/plotting/chart/CHART_CONFIG_ETHIOPIA_EPI_PINE.json",
+        "SysConfig" : "${ConfigsPath}/coordinator/sys_config_EastAfrica.json",
+        "RunConfig_seasonsofar" : "${ConfigsPath}/plotting/epi/RUN_CONFIG_EPI.json",
+        "RunConfig_seasonplusforecast" : "${ConfigsPath}/plotting/epi/RUN_CONFIG_EPI.json",
+        "ChartConfig" : "${ConfigsPath}/plotting/chart/CHART_CONFIG_ETHIOPIA_EPI_PINE.json",
         "PlottingRegionName" : "Ethiopia",
         "EpiCase" : "ps"
     }

pipeline_resources/configs/coordinator/survey_config_EastAfrica_fc_live.json

@@ -4,7 +4,7 @@
     "ProcessEWSPlotting": "process_EWS_plotting_survey",
     "AcceptableDowntimeDays": 70,
     "SeasonStartString": "20220815",
-    "ServerCredentialsFile": "${ResourcesPath}/configs/coordinator/Cred-WRT.json",
+    "ServerCredentialsFile": "${ConfigsPath}/coordinator/Cred-WRT.json",
     "Groups": {
         "PROD": {
             "Origin": [
@@ -72,7 +72,7 @@
     "SurveyorNameCol": "surveyor_infromation-surveyor_name",
     "SourcesRegionName": "EastAfrica",
     "SourcesConfigs": {
-        "WRT-Publ": "${ResourcesPath}/configs/source_gen/config_EastAfrica_mapspam2017.json"
+        "WRT-Publ": "${ConfigsPath}/source_gen/config_EastAfrica_mapspam2017.json"
     },
-    "SourcesConfigDefault": "${ResourcesPath}/configs/source_gen/config_EastAfrica_mapspam2017.json"
+    "SourcesConfigDefault": "${ConfigsPath}/source_gen/config_EastAfrica_mapspam2017.json"
 }
\ No newline at end of file

pipeline_resources/configs/coordinator/template_sys_config_EastAfrica.json

@@ -8,6 +8,7 @@
     "LoggingConfigPath" : "${ProjectRoot}/code/coordinator/configs/logger/template_log_config.json",
     "WorkspacePathout" : "${ProjectRoot}/regions/EastAfrica/workspace/",
     "WorkspacePath" : "${ProjectRoot}/regions/EastAfrica/workspace/",
-    "ResourcesPath" : "${ProjectRoot}/regions/EastAfrica/resources/",
+    "ConfigsPath" : "${ProjectRoot}/regions/EastAfrica/resources/configs",
+    "AssetsPath" : "<path to unpacked assets dir>",
     "PLACES_SHAPE_FILE": "${ProjectRoot}/regions/EastAfrica/resources/assets/plotting/gis_data/common/places_shape/east_africa_places.shp"
 }

pipeline_resources/configs/plotting/sys/SYS_CONFIG_PINE.json

-{
-}
\ No newline at end of file