From d3a4b6f3333dc55618ac162b3c1c3fe5ef2305c4 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Mon, 21 Aug 2023 15:51:40 +0100
Subject: [PATCH] changing the landpoint files to use a new column "included",
to filter for countries, rather than using the lat lon values to decide what
to filter
---
coordinator/EnvSuitPipeline.py | 28 +-
docs/_source/deployment_notes.rst | 111 ++++++++
docs/_source/installation.rst | 2 +
docs/api.rst | 2 +-
.../input_spatial_points_EastAfrica.csv | 254 +++++++++---------
.../east_africa/small_east_africa_points.txt | 2 +-
.../ethiopia/small_ethiopia_points.txt | 2 +-
7 files changed, 253 insertions(+), 148 deletions(-)
create mode 100644 docs/_source/deployment_notes.rst
diff --git a/coordinator/EnvSuitPipeline.py b/coordinator/EnvSuitPipeline.py
index 80d160f..f790a10 100644
--- a/coordinator/EnvSuitPipeline.py
+++ b/coordinator/EnvSuitPipeline.py
@@ -165,23 +165,6 @@ def run_pipeline(pipeline_config, region, dateString, extracted = False, prevent
region_outPath = os.path.join(outPath,'ENVIRONMENT_2.0_'+dateString,'processed',region)
if prevent_overwrite: assert not os.path.exists(region_outPath)
- # Get spatial points file for the region. The file template can be specified in the config.
- if "SPATIAL_POINTS_FILE_TEMPLATE" in pipeline_config:
- region_spatial_points_file_template = pipeline_config["SPATIAL_POINTS_FILE_TEMPLATE"]
- region_spatial_points_file = Template(region_spatial_points_file_template).substitute(Region=region)
- else:
- region_spatial_points_file = resourcesPath + 'assets/' + 'input_spatial_points_' + region + '.csv'
-
- input_spatial_points_file = workPath + 'input_spatial_points.csv'
- if prevent_overwrite: assert not os.path.exists(input_spatial_points_file)
- shutil.copy(region_spatial_points_file,input_spatial_points_file)
- """
- read in the input_spatial_points.csv file and get the number of spatial points which are marked as landpoint=True
- (we will be finltering the non-landpoint locations)
- """
- spatial_points = pd.read_csv(input_spatial_points_file)
- spatial_dim = spatial_points[spatial_points['landpoint']].shape[0]
-
# Generate extraction (input) and output temporal points files
timeresolution = 3 # hours
@@ -207,7 +190,7 @@ def run_pipeline(pipeline_config, region, dateString, extracted = False, prevent
# Modify run_config
config[ParamsFileParser.TIMEPOINTS_FILE_KEY] = extraction_temporal_points_file
config[ParamsFileParser.OUTPUT_DIR_KEY] = workPath
- config[ParamsFileParser.SPATIAL_POINTS_FILE_KEY] = input_spatial_points_file
+ # config[ParamsFileParser.SPATIAL_POINTS_FILE_KEY] = input_spatial_points_file
# note that this field will only get used by the ewa5_met_data_extraction code, which uses a multi-processor module
config[ParamsFileParser.MAX_PROCESSORS_KEY] = MAX_WORKERS
@@ -286,6 +269,15 @@ def run_pipeline(pipeline_config, region, dateString, extracted = False, prevent
result = pd.read_csv(resultFile)
result_dims = result.shape
+ """
+ read in the input_spatial_points.csv file and get the number of spatial points - this is used for
+ sense-checking the dimensions of the output
+ """
+ # todo we could swap the spatial points file for a file specifying the expected dimensions - much smaller
+ region_spatial_points_file = resourcesPath + 'assets/' + 'input_spatial_points_' + region + '.csv'
+ spatial_points = pd.read_csv(region_spatial_points_file)
+ spatial_dim = spatial_points.shape[0]
+
if ((result_dims[0] != spatial_dim) or (result_dims[1] != (temporal_dim + 4))): # + 4 required because there are extra columns in the result file
logger.error(f"Result dimension {result_dims} does not match with the expected: ({spatial_dim}, {temporal_dim + 4})")
raise IndexError
diff --git a/docs/_source/deployment_notes.rst b/docs/_source/deployment_notes.rst
new file mode 100644
index 0000000..2b6ed94
--- /dev/null
+++ b/docs/_source/deployment_notes.rst
@@ -0,0 +1,111 @@
+
+.. _deployment_notes:
+
+Deployment notes
+================
+
+The EWS pipeline uses an input grid to specify the points to be included in the calculations. The grid is determined by
+the UK Met Office (UKMO). The points specified within the NAME and weather data provided by the UKMO must be matched by
+the landpoint files specified below. The system used by the EWS uses a series of masks, to specify which points are over
+water, or outside country boundaries. The masking system allows faster calculation (avoiding areas that are not relevant)
+and plotting that only shows the regions of interest. The spatial grids used by the EWS pipeline must match the
+UKMO grid in number of points and sort order:
+
+.. code-block:: python
+
+ landpoint_df.sort_values(['longitude', 'latitude'], ascending = [True, True])
+
+The precision of the longitude and latitude values is not important, provided any differences in precision do not affect
+the sort order.
+
+Filtering is achieved using a column specifying whether it is "included" or not:
+
+.. code-block::
+
+ longitude,latitude,included
+ 21.445310,-5.015625,True
+ 21.445310,-4.921875,False
+ 21.445310,-4.828125,True
+ 21.445310,-4.734375,True
+
+The longitude and latitude areas are not technically necessary,
+as filtering is performed by building a series of true/false values to filter the incoming dataframe (hence the
+importance of the sort order and row count).
+
+.. code-block:: python
+
+ spatial_points_df = pd.read_csv(landpoint_file_path)
+ rows_to_keep = spatial_points_df['included']
+ filtered_df = unfiltered_df[rows_to_keep]
+
+
+Location of grid files
+======================
+
+Met extractor
+-------------
+
+The main input grid is stored in the assets folder of the met_extractor resources. e.g.
+
+**regions/<EastAfrica-SouthAsia>/met_extractor/assets/input_spatial_points_<EastAfrica-SouthAsia>.csv**
+
+.. admonition:: Configuration files from Cambridge required
+
+ This grid file is not used for masking, it is used for sense-checking the dimensions of output data. We could replace it
+ with the expected dimensions and extent of the input/output data.
+
+The file path is built within the code, so it must conform to the format:
+
+.. code-block:: python
+
+ 'input_spatial_points_' + region + '.csv'
+
+
+('region' is specified in the JSON config, e.g. config_EastAfrica_fc_live.json:
+
+.. code-block:: json
+
+ "RegionName" : "EastAfrica"
+
+Post processing
+---------------
+
+edit the "COUNTRY_LANDPOINTS_FILE" field in the chart config files used for post-processing e.g.
+
+regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_ETHIOPIA_PINE.json
+
+to use a grid that masks the points outside the country boundaries, stored in:
+
+/regions/EastAfrica/resources/plotting/assets/gis_data/<country>/<country>_points.txt
+
+The post-processing mask files are used in:
+
+:func:`ews_postprocessing.utils.processing_utils.mask_dataframe_for_landfile`
+
+this function is called from the Deposition, Environmental suitability and Epidemology pipelines to filter for country boundaries.
+
+Deposition: :meth:`~ews_postprocessing.deposition.condense_deposition_csvs.ConvertCondenseAndNormalizeDepositionCSV.mask_csv_for_land`
+
+Environmental suitability: :meth:`~ews_postprocessing.environmental_suitability.condense_environment_csvs.CondenseEnvSuitCSV.process`
+
+Epidemiology: :meth:`~ews_postprocessing.epi.epi_post_processor.EPIPostPostProcessor._mask_epi_csvs`
+
+
+Creating the grid files
+=======================
+
+A utility script:
+
+ews_postprocessing/utils/filter_country_points.py
+
+Can be used to create a set of mask tiles. It takes an ncdf file (provided by the UKMO) and writes a sorted list of lat
+lon values, with an "included" column, set to True where the point is within the shape file of interest.
+
+Defining the Met Data cube files
+================================
+
+edit "MET_DATA_TYPE": "NAME", field in the /regions/EastAfrica/resources/met_extractor/configs/template_operational_config.json file to match the correct met data lookup specified in:
+
+regions/EastAfrica/resources/met_extractor/configs/FIELD_NAME_CONSTANTS.csv
+
+this ensures the correct Met nc files are located
diff --git a/docs/_source/installation.rst b/docs/_source/installation.rst
index 9877901..31dec89 100644
--- a/docs/_source/installation.rst
+++ b/docs/_source/installation.rst
@@ -8,6 +8,8 @@ UK Met Office. These instructions will allow you to get the EWS system running o
data from the UK Met office (both weather data and the output of NAME runs. The Cambridge team can provide you with the
required configuration files and project assets for the regions currently being monitored.
+See also :ref:`deployment notes <deployment_notes>`
+
The example code below is from a Linux system using the Bash shell.
1: Create installation dirs:
diff --git a/docs/api.rst b/docs/api.rst
index 3d57073..92c6af1 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -6,7 +6,7 @@ API
:template: custom-module-template.rst
:recursive:
-.. coordinator
+ coordinator
plotting
ews_postprocessing
met_processing
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/met_extractor/assets/input_spatial_points_EastAfrica.csv b/tests/test_data/test_deployment/regions/EastAfrica/resources/met_extractor/assets/input_spatial_points_EastAfrica.csv
index b9136ae..ff5d243 100644
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/met_extractor/assets/input_spatial_points_EastAfrica.csv
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/met_extractor/assets/input_spatial_points_EastAfrica.csv
@@ -1,127 +1,127 @@
-longitude,latitude,landpoint
-38.60156,8.109375,True
-38.60156,8.203125,True
-38.60156,8.296875,True
-38.60156,8.390625,True
-38.60156,8.484375,True
-38.60156,8.578125,True
-38.60156,8.671875,True
-38.60156,8.765625,True
-38.60156,8.859375,True
-38.742185,8.109375,True
-38.742185,8.203125,True
-38.742185,8.296875,True
-38.742185,8.390625,True
-38.742185,8.484375,True
-38.742185,8.578125,True
-38.742185,8.671875,True
-38.742185,8.765625,True
-38.742185,8.859375,True
-38.88281,8.109375,True
-38.88281,8.203125,True
-38.88281,8.296875,True
-38.88281,8.390625,True
-38.88281,8.484375,True
-38.88281,8.578125,True
-38.88281,8.671875,True
-38.88281,8.765625,True
-38.88281,8.859375,True
-39.023435,8.109375,True
-39.023435,8.203125,True
-39.023435,8.296875,True
-39.023435,8.390625,True
-39.023435,8.484375,True
-39.023435,8.578125,True
-39.023435,8.671875,True
-39.023435,8.765625,True
-39.023435,8.859375,True
-39.16406,8.109375,True
-39.16406,8.203125,True
-39.16406,8.296875,True
-39.16406,8.390625,True
-39.16406,8.484375,True
-39.16406,8.578125,True
-39.16406,8.671875,True
-39.16406,8.765625,True
-39.16406,8.859375,True
-39.304685,8.109375,True
-39.304685,8.203125,True
-39.304685,8.296875,True
-39.304685,8.390625,True
-39.304685,8.484375,True
-39.304685,8.578125,True
-39.304685,8.671875,True
-39.304685,8.765625,True
-39.304685,8.859375,True
-39.44531,8.109375,True
-39.44531,8.203125,True
-39.44531,8.296875,True
-39.44531,8.390625,True
-39.44531,8.484375,True
-39.44531,8.578125,True
-39.44531,8.671875,True
-39.44531,8.765625,True
-39.44531,8.859375,True
-39.585935,8.109375,True
-39.585935,8.203125,True
-39.585935,8.296875,True
-39.585935,8.390625,True
-39.585935,8.484375,True
-39.585935,8.578125,True
-39.585935,8.671875,True
-39.585935,8.765625,True
-39.585935,8.859375,True
-39.72656,8.109375,True
-39.72656,8.203125,True
-39.72656,8.296875,True
-39.72656,8.390625,True
-39.72656,8.484375,True
-39.72656,8.578125,True
-39.72656,8.671875,True
-39.72656,8.765625,True
-39.72656,8.859375,True
-39.867185,8.109375,True
-39.867185,8.203125,True
-39.867185,8.296875,True
-39.867185,8.390625,True
-39.867185,8.484375,True
-39.867185,8.578125,True
-39.867185,8.671875,True
-39.867185,8.765625,True
-39.867185,8.859375,True
-40.00781,8.109375,True
-40.00781,8.203125,True
-40.00781,8.296875,True
-40.00781,8.390625,True
-40.00781,8.484375,True
-40.00781,8.578125,True
-40.00781,8.671875,True
-40.00781,8.765625,True
-40.00781,8.859375,True
-40.148435,8.109375,True
-40.148435,8.203125,True
-40.148435,8.296875,True
-40.148435,8.390625,True
-40.148435,8.484375,True
-40.148435,8.578125,True
-40.148435,8.671875,True
-40.148435,8.765625,True
-40.148435,8.859375,True
-40.28906,8.109375,True
-40.28906,8.203125,True
-40.28906,8.296875,True
-40.28906,8.390625,True
-40.28906,8.484375,True
-40.28906,8.578125,True
-40.28906,8.671875,True
-40.28906,8.765625,True
-40.28906,8.859375,True
-40.429685,8.109375,True
-40.429685,8.203125,True
-40.429685,8.296875,True
-40.429685,8.390625,True
-40.429685,8.484375,True
-40.429685,8.578125,True
-40.429685,8.671875,True
-40.429685,8.765625,True
-40.429685,8.859375,True
+longitude,latitude
+38.60156,8.109375
+38.60156,8.203125
+38.60156,8.296875
+38.60156,8.390625
+38.60156,8.484375
+38.60156,8.578125
+38.60156,8.671875
+38.60156,8.765625
+38.60156,8.859375
+38.742185,8.109375
+38.742185,8.203125
+38.742185,8.296875
+38.742185,8.390625
+38.742185,8.484375
+38.742185,8.578125
+38.742185,8.671875
+38.742185,8.765625
+38.742185,8.859375
+38.88281,8.109375
+38.88281,8.203125
+38.88281,8.296875
+38.88281,8.390625
+38.88281,8.484375
+38.88281,8.578125
+38.88281,8.671875
+38.88281,8.765625
+38.88281,8.859375
+39.023435,8.109375
+39.023435,8.203125
+39.023435,8.296875
+39.023435,8.390625
+39.023435,8.484375
+39.023435,8.578125
+39.023435,8.671875
+39.023435,8.765625
+39.023435,8.859375
+39.16406,8.109375
+39.16406,8.203125
+39.16406,8.296875
+39.16406,8.390625
+39.16406,8.484375
+39.16406,8.578125
+39.16406,8.671875
+39.16406,8.765625
+39.16406,8.859375
+39.304685,8.109375
+39.304685,8.203125
+39.304685,8.296875
+39.304685,8.390625
+39.304685,8.484375
+39.304685,8.578125
+39.304685,8.671875
+39.304685,8.765625
+39.304685,8.859375
+39.44531,8.109375
+39.44531,8.203125
+39.44531,8.296875
+39.44531,8.390625
+39.44531,8.484375
+39.44531,8.578125
+39.44531,8.671875
+39.44531,8.765625
+39.44531,8.859375
+39.585935,8.109375
+39.585935,8.203125
+39.585935,8.296875
+39.585935,8.390625
+39.585935,8.484375
+39.585935,8.578125
+39.585935,8.671875
+39.585935,8.765625
+39.585935,8.859375
+39.72656,8.109375
+39.72656,8.203125
+39.72656,8.296875
+39.72656,8.390625
+39.72656,8.484375
+39.72656,8.578125
+39.72656,8.671875
+39.72656,8.765625
+39.72656,8.859375
+39.867185,8.109375
+39.867185,8.203125
+39.867185,8.296875
+39.867185,8.390625
+39.867185,8.484375
+39.867185,8.578125
+39.867185,8.671875
+39.867185,8.765625
+39.867185,8.859375
+40.00781,8.109375
+40.00781,8.203125
+40.00781,8.296875
+40.00781,8.390625
+40.00781,8.484375
+40.00781,8.578125
+40.00781,8.671875
+40.00781,8.765625
+40.00781,8.859375
+40.148435,8.109375
+40.148435,8.203125
+40.148435,8.296875
+40.148435,8.390625
+40.148435,8.484375
+40.148435,8.578125
+40.148435,8.671875
+40.148435,8.765625
+40.148435,8.859375
+40.28906,8.109375
+40.28906,8.203125
+40.28906,8.296875
+40.28906,8.390625
+40.28906,8.484375
+40.28906,8.578125
+40.28906,8.671875
+40.28906,8.765625
+40.28906,8.859375
+40.429685,8.109375
+40.429685,8.203125
+40.429685,8.296875
+40.429685,8.390625
+40.429685,8.484375
+40.429685,8.578125
+40.429685,8.671875
+40.429685,8.765625
+40.429685,8.859375
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/plotting/assets/gis_data/east_africa/small_east_africa_points.txt b/tests/test_data/test_deployment/regions/EastAfrica/resources/plotting/assets/gis_data/east_africa/small_east_africa_points.txt
index b9136ae..720925f 100644
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/plotting/assets/gis_data/east_africa/small_east_africa_points.txt
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/plotting/assets/gis_data/east_africa/small_east_africa_points.txt
@@ -1,4 +1,4 @@
-longitude,latitude,landpoint
+longitude,latitude,included
38.60156,8.109375,True
38.60156,8.203125,True
38.60156,8.296875,True
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/plotting/assets/gis_data/ethiopia/small_ethiopia_points.txt b/tests/test_data/test_deployment/regions/EastAfrica/resources/plotting/assets/gis_data/ethiopia/small_ethiopia_points.txt
index 040c41b..8282fd5 100644
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/plotting/assets/gis_data/ethiopia/small_ethiopia_points.txt
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/plotting/assets/gis_data/ethiopia/small_ethiopia_points.txt
@@ -1,4 +1,4 @@
-longitude,latitude,landpoint
+longitude,latitude,included
38.60156,8.109375,True
38.60156,8.203125,True
38.60156,8.296875,True
--
GitLab