From 5eb92fe3b12ad174691eb381802207bec48ab701 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Fri, 28 Oct 2022 15:12:21 +0100
Subject: [PATCH] adding first proper asserts to depo test
---
tests/integration/integration_test_utils.py | 7 ++
tests/integration/test_deposition.py | 97 +++++++++++++++----
.../configs/config_EastAfrica_fc_live.json | 2 +-
3 files changed, 86 insertions(+), 20 deletions(-)
diff --git a/tests/integration/integration_test_utils.py b/tests/integration/integration_test_utils.py
index 5306825..26d0aeb 100644
--- a/tests/integration/integration_test_utils.py
+++ b/tests/integration/integration_test_utils.py
@@ -1,3 +1,4 @@
+import glob
import json
from datetime import datetime
from zipfile import ZipFile
@@ -5,6 +6,7 @@ from zipfile import ZipFile
class IntegrationTestUtils:
+ DEFAULT_CONFIG_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json"
TEMP_CONFIG_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/workspace/temp_config.json"
EXAMPLE_SURVEY_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_survey_run.zip"
EXAMPLE_DEPO_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_depo_run.zip"
@@ -34,3 +36,8 @@ class IntegrationTestUtils:
with ZipFile(zip_to_unpack) as zf: # open the zip file
for target_file in zf.namelist(): # check if the file exists in the archive
zf.extract(target_file, out_file)
+
+ @staticmethod
+ def count_files_in_wildcard(wildcard: str) -> int:
+ results = glob.glob(wildcard)
+ return len(results)
diff --git a/tests/integration/test_deposition.py b/tests/integration/test_deposition.py
index 48508af..36a2ef1 100644
--- a/tests/integration/test_deposition.py
+++ b/tests/integration/test_deposition.py
@@ -6,42 +6,101 @@ from integration.integration_test_utils import IntegrationTestUtils
class TestDeposition(unittest.TestCase):
+ TEST_OUT_PATH: str = "not_set"
+ TEST_RUN_DATE: str = '20221001'
+ TEST_JOB_DIR: str = "DEPOSITION_" + TEST_RUN_DATE
- def setUp(self) -> None:
- super().setUp()
- default_config = '../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json'
- self.default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
- i = 0
- def test_depo_standard_inputs_expected_results1(self):
+ @classmethod
+ def setUpClass(cls) -> None:
+ TestDeposition.write_temp_run_config_file()
+ TestDeposition.run_depo_pipelines()
+
+ @staticmethod
+ def write_temp_run_config_file():
nowstring: str = IntegrationTestUtils.get_now_string()
+ prefix: str = "temp_depo_" + nowstring
+ # prefix: str = ""
+
+ default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
+ default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
+ run_dict: dict = copy.deepcopy(default_config_dict)
+ TestDeposition.TEST_OUT_PATH = run_dict['WorkspacePathout'] + prefix + os.sep
+ run_dict['WorkspacePathout'] = TestDeposition.TEST_OUT_PATH
+ run_dict['WorkspacePath'] = TestDeposition.TEST_OUT_PATH
+ run_dict['ServerName'] = '' # nothing, as local machine
+ full_server_path = os.path.abspath("../test_data/met_office/depo/")
+ run_dict['Deposition']['ServerPathTemplate'] = full_server_path
+
+ IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
+
+ @staticmethod
+ def run_depo_pipelines():
os.environ["EMAIL_CRED"] = "../test_data/test_deployment/envs/Cred_gmail.json"
from Processor import run_Process, set_log_level
+
+
args_dict: dict = {}
args_dict['component'] = 'Deposition'
args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH]
args_dict['log_level'] = 'error'
args_dict['live'] = False
- args_dict['start_date'] = '20221001'
+ args_dict['start_date'] = TestDeposition.TEST_RUN_DATE
args_dict['noupload'] = True
set_log_level(args_dict['log_level'])
- run_dict: dict = copy.deepcopy(self.default_config_dict)
- test_out_path = run_dict['WorkspacePathout'] + "temp_depo_" + nowstring + os.sep
- run_dict['WorkspacePathout'] = test_out_path
- run_dict['WorkspacePath'] = test_out_path
- run_dict['ServerName'] = '' # nothing, as local machine
- full_server_path = os.path.abspath("../test_data/met_office/depo/")
- run_dict['Deposition']['ServerPathTemplate'] = full_server_path
+ run_Process(args_dict)
- # run_dict['SubRegionNames'].remove('Kenya')
- # run_dict['Deposition']['EWS-Plotting'].pop('Kenya', None)
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
- run_Process(args_dict)
- self.assertTrue(True)
+ def test_standard_run_input_status_success(self):
+ status_file_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR, "STATUS_SUCCESS")
+ success_file_exists: bool = os.path.isfile(status_file_path)
+ self.assertTrue(success_file_exists)
+
+
+ def test_standard_run_input_all_regions_ran(self):
+ """
+ working on the assumption that if there are images for each region, it must have run through
+ (at least past the region iteration)
+ """
+
+ east_africa_image_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR,
+ "plotting", "eastafrica", "images", "Weekly",
+ "deposition_eastafrica_leaf_rust_total_202210010000_202210080000_map.png")
+ ethiopia_image_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR,
+ "plotting", "ethiopia", "images", "Weekly",
+ "deposition_ethiopia_leaf_rust_total_202210010000_202210080000_map.png")
+
+ ea_file_exists: bool = os.path.isfile(east_africa_image_path)
+ eth_file_exists: bool = os.path.isfile(ethiopia_image_path)
+ self.assertTrue(ea_file_exists)
+ self.assertTrue(eth_file_exists)
+
+
+ def test_standard_run_all_input_csvs_produced(self):
+ east_africa_csv_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR,
+ "plotting", "eastafrica", "input_csvs", "*.csv")
+ ethiopia_csv_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR,
+ "plotting", "ethiopia", "input_csvs", "*.csv")
+
+ ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_csv_path)
+ eth_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(ethiopia_csv_path)
+ self.assertEqual(9, ea_csv_count)
+ self.assertEqual(27, eth_csv_count)
+
+
+ def test_standard_run_all_images_produced(self):
+ east_africa_image_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR,
+ "plotting", "eastafrica", "images", "Weekly", "*.png")
+ ethiopia_image_path = os.path.join(TestDeposition.TEST_OUT_PATH, TestDeposition.TEST_JOB_DIR,
+ "plotting", "ethiopia", "images", "Weekly", "*.png")
+
+ ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_image_path)
+ eth_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(ethiopia_image_path)
+ self.assertEqual(3, ea_csv_count)
+ self.assertEqual(6, eth_csv_count)
if __name__ == '__main__':
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
index e74901c..12cf382 100644
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
@@ -4,7 +4,7 @@
"StartTime" : "?",
"StartString" : "?",
"WorkspacePathout" : "../test_data/test_deployment/regions/EastAfrica/workspace/",
- "WorkspacePath" : "../test_data/test_deployment/regions/EastAfrica/workspace/",
+ "WorkspacePath" : "set_in_the_code",
"ResourcesPath" : "../test_data/test_deployment/regions/EastAfrica/resources/",
"ServerPath" : "/storage/moved/Ethiopia/",
"ServerName" : "ewsmanager@willow.csx.cam.ac.uk",
--
GitLab