From 40e382cc6ca96bb694758e6cd3ddfcc1ae453f7e Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Thu, 30 Nov 2023 12:31:50 +0000
Subject: [PATCH] groundwork for using tifs/netcdf throughout the pipeline.
splitting out some csv parsing functions
---
coordinator/EnvSuitPipeline.py | 14 +++++++-------
coordinator/ProcessorEnvironment.py | 5 +++--
tests/integration/partial/test_env_suit.py | 4 ++--
3 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/coordinator/EnvSuitPipeline.py b/coordinator/EnvSuitPipeline.py
index daf5e3a..20e570c 100644
--- a/coordinator/EnvSuitPipeline.py
+++ b/coordinator/EnvSuitPipeline.py
@@ -261,13 +261,13 @@ def run_pipeline(pipeline_config, region, dateString, extracted = False, prevent
# run_merger(config, sys_config, processor_name)
- resultFile = envSuitPath + 'RIE.csv'
+ resultFile = envSuitPath + 'RIE.tif'
strain_outPath = os.path.join(region_outPath,strain)
- strain_outFile = strain_outPath + '/RIE_value.csv'
+ strain_outFile = strain_outPath + '/RIE_value.tif'
# Check results dimension
- result = pd.read_csv(resultFile)
- result_dims = result.shape
+ # result = pd.read_csv(resultFile)
+ # result_dims = result.shape
"""
read in the input_spatial_points.csv file and get the number of spatial points - this is used for
@@ -278,9 +278,9 @@ def run_pipeline(pipeline_config, region, dateString, extracted = False, prevent
spatial_points = pd.read_csv(region_spatial_points_file)
spatial_dim = spatial_points.shape[0]
- if ((result_dims[0] != spatial_dim) or (result_dims[1] != (temporal_dim + 4))): # + 4 required because there are extra columns in the result file
- logger.error(f"Result dimension {result_dims} does not match with the expected: ({spatial_dim}, {temporal_dim + 4})")
- raise IndexError
+ # if ((result_dims[0] != spatial_dim) or (result_dims[1] != (temporal_dim + 4))): # + 4 required because there are extra columns in the result file
+ # logger.error(f"Result dimension {result_dims} does not match with the expected: ({spatial_dim}, {temporal_dim + 4})")
+ # raise IndexError
if not os.path.exists(strain_outPath):
os.makedirs(strain_outPath)
diff --git a/coordinator/ProcessorEnvironment.py b/coordinator/ProcessorEnvironment.py
index df64fc0..c90530b 100644
--- a/coordinator/ProcessorEnvironment.py
+++ b/coordinator/ProcessorEnvironment.py
@@ -180,8 +180,9 @@ class ProcessorEnvironment(Processor):
for region in subregions:
output_dir = f"{jobPath}/plotting/{region.lower()}"
- csv_template_dir = input_dir + "/{DISEASE_DIR}/RIE_value.csv"
-
+ # csv_template_dir = input_dir + "/{DISEASE_DIR}/RIE_value.csv"
+ csv_template_dir = input_dir + "/{DISEASE_DIR}/RIE_value.tif"
+
Path(output_dir).mkdir(parents=True, exist_ok=True)
sys_config = config['Environment']['EWS-Plotting']['SysConfig']
diff --git a/tests/integration/partial/test_env_suit.py b/tests/integration/partial/test_env_suit.py
index 11169a6..0c5c44e 100644
--- a/tests/integration/partial/test_env_suit.py
+++ b/tests/integration/partial/test_env_suit.py
@@ -42,8 +42,8 @@ class TestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
@staticmethod
def write_temp_run_config_file():
nowstring: str = IntegrationTestUtils.get_now_string()
- prefix: str = "temp_env_" + nowstring
- #prefix: str = "temp_env"
+ # prefix: str = "temp_env_" + nowstring
+ prefix: str = "temp_env"
default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
--
GitLab