changing the optional SPATIAL_POINTS_FILE config parameter to...

changing the optional SPATIAL_POINTS_FILE config parameter to SPATIAL_POINTS_FILE_TEMPLATE and allowing ${} value subsiitution in the paremeter value
removing some points from the ethiopia post processing landpoint file, to allow checking that the filtering for post-processing is working

coordinator/EnvSuitPipeline.py

@@ -5,6 +5,7 @@ import json
 import logging
 import os
 import shutil
+from string import Template
 
 import pandas as pd
 
@@ -164,9 +165,10 @@ def run_pipeline(pipeline_config, region, dateString, extracted = False, prevent
     region_outPath = os.path.join(outPath,'ENVIRONMENT_2.0_'+dateString,'processed',region)
     if prevent_overwrite: assert not os.path.exists(region_outPath)
 
-    # Get spatial points file for the region. The file can be calculated from the region name or specified in the config.
-    if "SPATIAL_POINTS_FILE" in pipeline_config:
-        region_spatial_points_file = pipeline_config["SPATIAL_POINTS_FILE"]
+    # Get spatial points file for the region. The file template can be specified in the config.
+    if "SPATIAL_POINTS_FILE_TEMPLATE" in pipeline_config:
+        region_spatial_points_file_template = pipeline_config["SPATIAL_POINTS_FILE_TEMPLATE"]
+        region_spatial_points_file = Template(region_spatial_points_file_template).substitute(Region=region)
     else:
         region_spatial_points_file = resourcesPath + 'assets/' + 'input_spatial_points_' + region + '.csv'
 

coordinator/ProcessorEnvironment.py

@@ -67,7 +67,8 @@ def process_in_job_env2_0(jobPath,status,config,component):
     cube_wildcard = f"{output_directory}/*.nc"
     cubes: CubeList = iris.load(cube_wildcard)
 
-    #TODO: find out from Will whether these problem cubes with unexpected time counts will be in the production output
+    # land_fraction and topography will only have a single timepoint (as these dont change over time), so we can ignore
+    # these when sense-checking the expected number of timepoints
     ignore_list = ["LAND_FRACTION", "TOPOGRAPHY"]
 
     for cube in cubes:

tests/test_data/test_deployment/regions/EastAfrica/resources/plotting/assets/gis_data/ethiopia/small_ethiopia_points.txt

@@ -97,20 +97,6 @@ longitude,latitude
 40.148435,8.671875
 40.28906,8.671875
 40.429685,8.671875
-38.60156,8.765625
-38.742185,8.765625
-38.88281,8.765625
-39.023435,8.765625
-39.16406,8.765625
-39.304685,8.765625
-39.44531,8.765625
-39.585935,8.765625
-39.72656,8.765625
-39.867185,8.765625
-40.00781,8.765625
-40.148435,8.765625
-40.28906,8.765625
-40.429685,8.765625
 38.60156,8.859375
 38.742185,8.859375
 38.88281,8.859375