groundwork for using tifs/netcdf throughout the pipeline.

splitting out some csv parsing functions

coordinator/EnvSuitPipeline.py

@@ -261,13 +261,13 @@ def run_pipeline(pipeline_config, region, dateString, extracted = False, prevent
 
                 # run_merger(config, sys_config, processor_name)
 
-                resultFile = envSuitPath + 'RIE.csv'
+                resultFile = envSuitPath + 'RIE.tif'
                 strain_outPath = os.path.join(region_outPath,strain)
-                strain_outFile = strain_outPath + '/RIE_value.csv'
+                strain_outFile = strain_outPath + '/RIE_value.tif'
 
                 # Check results dimension
-                result = pd.read_csv(resultFile)
-                result_dims = result.shape
+                # result = pd.read_csv(resultFile)
+                # result_dims = result.shape
 
                 """
                 read in the input_spatial_points.csv file and get the number of spatial points - this is used for 
@@ -278,9 +278,9 @@ def run_pipeline(pipeline_config, region, dateString, extracted = False, prevent
                 spatial_points = pd.read_csv(region_spatial_points_file)
                 spatial_dim = spatial_points.shape[0]
 
-                if ((result_dims[0] != spatial_dim) or (result_dims[1] != (temporal_dim + 4))): # + 4 required because there are extra columns in the result file
-                    logger.error(f"Result dimension {result_dims} does not match with the expected: ({spatial_dim}, {temporal_dim + 4})")
-                    raise IndexError
+                # if ((result_dims[0] != spatial_dim) or (result_dims[1] != (temporal_dim + 4))): # + 4 required because there are extra columns in the result file
+                #     logger.error(f"Result dimension {result_dims} does not match with the expected: ({spatial_dim}, {temporal_dim + 4})")
+                #     raise IndexError
 
                 if not os.path.exists(strain_outPath):
                     os.makedirs(strain_outPath)

coordinator/ProcessorEnvironment.py

@@ -180,8 +180,9 @@ class ProcessorEnvironment(Processor):
         for region in subregions:
     
             output_dir = f"{jobPath}/plotting/{region.lower()}"
-            csv_template_dir = input_dir + "/{DISEASE_DIR}/RIE_value.csv"
-    
+            # csv_template_dir = input_dir + "/{DISEASE_DIR}/RIE_value.csv"
+            csv_template_dir = input_dir + "/{DISEASE_DIR}/RIE_value.tif"
+
             Path(output_dir).mkdir(parents=True, exist_ok=True)
     
             sys_config = config['Environment']['EWS-Plotting']['SysConfig']

tests/integration/partial/test_env_suit.py

@@ -42,8 +42,8 @@ class TestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
     @staticmethod
     def write_temp_run_config_file():
         nowstring: str = IntegrationTestUtils.get_now_string()
-        prefix: str = "temp_env_" + nowstring
-        #prefix: str = "temp_env"
+        # prefix: str = "temp_env_" + nowstring
+        prefix: str = "temp_env"
 
         default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
         default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)