From da5d6f8a0853b3345da78b2174a40d36efe1df72 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Thu, 23 Mar 2023 11:55:48 +0000
Subject: [PATCH] trying to get full-fat epi test to run on server
---
.gitlab-ci.yml | 33 +++++
tests/integration/full/full_test_epi.py | 113 ++++++++++++++++++
tests/integration/partial/test_epi.py | 113 +++++-------------
.../test_suites/depo_test_suite.py | 6 -
.../test_suites/env_suit_test_suite.py | 6 -
.../test_suites/survey_test_suite.py | 6 -
6 files changed, 175 insertions(+), 102 deletions(-)
create mode 100644 tests/integration/full/full_test_epi.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b15f4c4..dba709a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -261,3 +261,36 @@ full_fat_advisory:
- $OUTPUT_DIR/*/log.txt
- $OUTPUT_DIR/*.txt
expire_in: 10 days
+
+full_fat_epi:
+ when: manual
+ stage: full-fat
+ extends: .before_script
+ image: lb584/ews_coordinator:latest
+ tags: ["ewas-production"]
+ script:
+ - >
+ cd $CI_PROJECT_DIR/tests/integration/full/
+
+ python3 full_test_epi.py
+ --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+ --outdir $OUTPUT_DIR
+ --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
+ --run_date_type $FULL_FAT_RUN_DATE
+ --custom_run_date $CUSTOM_RUN_DATE
+
+ after_script:
+ - >
+ if [[ "$FULL_FAT_COPY_OUTPUT" == "true" ]];
+ then
+ date=$(date '+%Y-%m-%d-%H%M');
+ mkdir ${TEST_WORKSPACE_DIR}/integration/full/${date};
+ cp -r $OUTPUT_DIR/* ${TEST_WORKSPACE_DIR}/integration/full/${date};
+ fi
+
+ artifacts:
+ when: on_failure
+ paths:
+ - $OUTPUT_DIR/*/log.txt
+ - $OUTPUT_DIR/*.txt
+ expire_in: 10 days
diff --git a/tests/integration/full/full_test_epi.py b/tests/integration/full/full_test_epi.py
new file mode 100644
index 0000000..d473612
--- /dev/null
+++ b/tests/integration/full/full_test_epi.py
@@ -0,0 +1,113 @@
+import argparse
+import copy
+import os
+import sys
+import unittest
+
+from integration.partial.integration_test_utils import IntegrationTestUtils
+from integration.test_suites.epi_test_suite import BaseEpiTestSuite
+
+
+class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
+
+ def set_expected_values(self):
+ # nothing to override here (outpouts overridden in the __main__ function
+ pass
+
+
+ def setUp(self) -> None:
+
+ self.set_expected_values()
+
+ # if BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH is None or not os.path.isdir(BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH):
+ if True:
+ FullTestEpi.write_temp_run_config_files()
+ # FullTestEpi.run_dependent_pipelines()
+ FullTestEpi.run_epi_pipeline()
+ else:
+ print(f"output in {BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH} already written, skipping rerun")
+
+
+ @staticmethod
+ def write_temp_run_config_files():
+ default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
+ default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
+ run_dict: dict = copy.deepcopy(default_config_dict)
+ run_dict['WorkspacePathout'] = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH
+ run_dict['WorkspacePath'] = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH
+ # we need to run the env suit pipeline as well
+ run_dict['Environment']['WORK_PATH'] = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH
+ run_dict['Environment']['INPUT_PATH'] = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH
+ run_dict['Environment']['OUTPUT_PATH'] = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH
+ #make sure we activate server download
+ run_dict['Survey']['SkipServerDownload'] = False
+
+ # change the config for the Epi so it only runs on a single past date. Requires use of a different
+ # FileListerFunction and a different calculation span
+ run_dict['Epidemiology']['CalculationSpanDays'] = [0, 1]
+ run_dict['Epidemiology']['Deposition']['FileListerFunction'] = "list_onefile_operational"
+ run_dict['Epidemiology']['Environment']['FileListerFunction'] = "list_onefile_operational"
+ # run_dict['Epidemiology']['Epi']['rescale_output_by_host_raster'] = False
+ previous_day_string: str = IntegrationTestUtils.get_day_before_as_string(BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE)
+ run_dict['Survey']['SeasonStartString'] = previous_day_string
+
+ # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
+ os.makedirs(BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH, exist_ok = True)
+ IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
+
+
+ @staticmethod
+ def run_dependent_pipelines():
+ IntegrationTestUtils.run_external_pipeline("Deposition",
+ BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE,
+ IntegrationTestUtils.EMAIL_CRED_PATH)
+ IntegrationTestUtils.run_external_pipeline("Environment",
+ BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE,
+ IntegrationTestUtils.EMAIL_CRED_PATH)
+
+ previous_day_string: str = IntegrationTestUtils.get_day_before_as_string(BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE)
+ IntegrationTestUtils.run_external_pipeline("Survey",
+ previous_day_string,
+ IntegrationTestUtils.EMAIL_CRED_PATH)
+ pass
+
+ @staticmethod
+ def run_epi_pipeline():
+ component = 'Epidemiology'
+ IntegrationTestUtils.run_external_pipeline(component,
+ BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE,
+ IntegrationTestUtils.EMAIL_CRED_PATH)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--config', required = True)
+ parser.add_argument('--outdir', required = True)
+ parser.add_argument('--email_cred', required = True)
+ parser.add_argument('--run_date_type', required = False)
+ parser.add_argument('--custom_run_date', required = False)
+ parser.add_argument('unittest_args', nargs='*')
+
+ _args = parser.parse_args()
+ _config_file: str = _args.config
+ _outdir: str = _args.outdir
+ _email_cred_path: str = _args.email_cred
+ _run_date_type: str = _args.run_date_type
+ _custom_run_date: str = _args.custom_run_date
+
+ IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH = _config_file
+
+ nowstring: str = IntegrationTestUtils.get_now_string()
+ # prefix: str = "temp_epi_" + nowstring
+ prefix: str = "temp_epi"
+ BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH = _outdir + prefix + os.sep
+ IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path
+
+ BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE = IntegrationTestUtils.generate_run_date(_run_date_type, _custom_run_date)
+
+ BaseEpiTestSuite.EpiTestSuite.TEST_JOB_DIR = "EPI_" + BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE
+
+ # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
+ sys.argv[1:] = _args.unittest_args
+ unittest.main()
diff --git a/tests/integration/partial/test_epi.py b/tests/integration/partial/test_epi.py
index 4707510..bddbc33 100644
--- a/tests/integration/partial/test_epi.py
+++ b/tests/integration/partial/test_epi.py
@@ -3,19 +3,26 @@ import os
import unittest
from integration.partial.integration_test_utils import IntegrationTestUtils
+from integration.test_suites.epi_test_suite import BaseEpiTestSuite
-class TestEpi(unittest.TestCase):
+class TestEpi(BaseEpiTestSuite.EpiTestSuite):
- TEST_OUT_PATH: str = "not_set"
- TEST_START_DATE: str = '20221001'
- TEST_JOB_DIR: str = "EPI_" + TEST_START_DATE
+ def set_expected_values(self):
+ BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE = '20221001'
+ BaseEpiTestSuite.EpiTestSuite.TEST_JOB_DIR = "EPI_" + BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE
- @classmethod
- def setUpClass(cls) -> None:
- TestEpi.write_temp_run_config_files()
- TestEpi.unpack_dependencies()
- TestEpi.run_epi_pipeline()
+
+ def setUp(self) -> None:
+
+ self.set_expected_values()
+
+ if BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH is None or not os.path.isdir(BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH):
+ TestEpi.write_temp_run_config_files()
+ TestEpi.unpack_dependencies()
+ TestEpi.run_epi_pipeline()
+ else:
+ print(f"output in {BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH} already written, skipping rerun")
@staticmethod
@@ -27,94 +34,32 @@ class TestEpi(unittest.TestCase):
default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
run_dict: dict = copy.deepcopy(default_config_dict)
- TestEpi.TEST_OUT_PATH = IntegrationTestUtils.TEST_WORKSPACE_PATH + prefix + os.sep
- run_dict['WorkspacePathout'] = TestEpi.TEST_OUT_PATH
- run_dict['WorkspacePath'] = TestEpi.TEST_OUT_PATH
+ BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH = IntegrationTestUtils.TEST_WORKSPACE_PATH + prefix + os.sep
+ run_dict['WorkspacePathout'] = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH
+ run_dict['WorkspacePath'] = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH
run_dict['ServerName'] = '' # nothing, as local machine
+ run_dict['Epidemiology']['CalculationSpanDays'] = [0, 1]
- os.makedirs(TestEpi.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = TestEpi.TEST_OUT_PATH + "temp_config.json"
+ os.makedirs(BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH, exist_ok = True)
+ IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH + "temp_config.json"
name = IntegrationTestUtils.TEMP_CONFIG_FILE_NAME
IntegrationTestUtils.write_json_file(run_dict, name)
@staticmethod
def unpack_dependencies():
- IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_SURVEY_FILE_PATH, TestEpi.TEST_OUT_PATH)
- IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_DEPO_FILE_PATH, TestEpi.TEST_OUT_PATH)
- IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_ENV_SUIT_FILE_PATH, TestEpi.TEST_OUT_PATH)
+ IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_SURVEY_FILE_PATH,
+ BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH)
+ IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_DEPO_FILE_PATH,
+ BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH)
+ IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_ENV_SUIT_FILE_PATH,
+ BaseEpiTestSuite.EpiTestSuite.TEST_OUT_PATH)
@staticmethod
def run_epi_pipeline():
component = 'Epidemiology'
- IntegrationTestUtils.run_unittest_pipeline(component, TestEpi.TEST_START_DATE)
-
-
- def test_standard_run_input_status_success(self):
- status_file_path = os.path.join(TestEpi.TEST_OUT_PATH, TestEpi.TEST_JOB_DIR, "STATUS_SUCCESS")
- success_file_exists: bool = os.path.isfile(status_file_path)
- self.assertTrue(success_file_exists)
-
- def test_standard_run_input_all_stem_rust_files_produced(self):
-
- alpha_beta_gamma_root = os.path.join(TestEpi.TEST_OUT_PATH, TestEpi.TEST_JOB_DIR,
- "EastAfrica",
- "StemRust",
- "*psbeta0.004gamma0.00025alpha1.0")
-
- self.check_wildcard_exists_and_not_empty(alpha_beta_gamma_root, ".csv")
- self.check_wildcard_exists_and_not_empty(alpha_beta_gamma_root, "*comparison.png")
- self.check_wildcard_exists_and_not_empty(alpha_beta_gamma_root, "*progression.csv")
- self.check_wildcard_exists_and_not_empty(alpha_beta_gamma_root, ".png")
- self.check_wildcard_exists_and_not_empty(alpha_beta_gamma_root, ".tif")
-
- env_root = os.path.join(TestEpi.TEST_OUT_PATH, TestEpi.TEST_JOB_DIR,
- "EastAfrica",
- "StemRust",
- "infections_temp_config_Epidemiology")
-
- self.check_wildcard_exists_and_not_empty(env_root, "*env.csv")
- self.check_wildcard_exists_and_not_empty(env_root, "*env.png")
- self.check_wildcard_exists_and_not_empty(env_root, "*env.tif")
-
- data_input_root = os.path.join(TestEpi.TEST_OUT_PATH, TestEpi.TEST_JOB_DIR,
- "EastAfrica",
- "StemRust",
- "input_data",
- "data_input*")
-
- self.check_wildcard_exists_and_not_empty(data_input_root, "*deposition.csv")
- self.check_wildcard_exists_and_not_empty(data_input_root, "*environment.csv")
-
- map_spam_root = os.path.join(TestEpi.TEST_OUT_PATH, TestEpi.TEST_JOB_DIR,
- "EastAfrica",
- "StemRust",
- "input_data",
- "wheat_area_frac_MapSPAM2010_EastAfrica*")
-
- self.check_wildcard_exists_and_not_empty(map_spam_root, ".csv")
- self.check_wildcard_exists_and_not_empty(map_spam_root, ".csv")
-
- def test_standard_run_input_all_plotting_files_produced(self):
-
- input_csvs_root = os.path.join(TestEpi.TEST_OUT_PATH, TestEpi.TEST_JOB_DIR,
- "plotting",
- "input_csvs",
- "*stem*")
-
- self.check_wildcard_exists_and_not_empty(input_csvs_root, ".csv")
-
- images_root = os.path.join(TestEpi.TEST_OUT_PATH, TestEpi.TEST_JOB_DIR,
- "plotting",
- "images",
- "*stem*")
-
- self.check_wildcard_exists_and_not_empty(images_root, "png")
-
- def check_wildcard_exists_and_not_empty(self, root: str, extension: str):
- file_exists: bool = IntegrationTestUtils.check_wildcard_exists_and_not_empty(root + extension)
- self.assertTrue(file_exists)
+ IntegrationTestUtils.run_unittest_pipeline(component, BaseEpiTestSuite.EpiTestSuite.TEST_START_DATE)
if __name__ == '__main__':
diff --git a/tests/integration/test_suites/depo_test_suite.py b/tests/integration/test_suites/depo_test_suite.py
index d8f4c39..d952ebf 100644
--- a/tests/integration/test_suites/depo_test_suite.py
+++ b/tests/integration/test_suites/depo_test_suite.py
@@ -8,12 +8,6 @@ from integration.partial.integration_test_utils import IntegrationTestUtils
class BaseDepoTestSuite:
- """
- to prevent the full integrations tests that extend the partial integrations tests running both sets of tests,
- the test suite is nested in a base class, to keep it at a separate package-level
- https://stackoverflow.com/questions/1323455/python-unit-test-with-base-and-sub-class
- """
-
class DepoTestSuite(unittest.TestCase):
TEST_OUT_PATH: str = None
diff --git a/tests/integration/test_suites/env_suit_test_suite.py b/tests/integration/test_suites/env_suit_test_suite.py
index 9253484..08c4dc3 100644
--- a/tests/integration/test_suites/env_suit_test_suite.py
+++ b/tests/integration/test_suites/env_suit_test_suite.py
@@ -8,12 +8,6 @@ from integration.partial.integration_test_utils import IntegrationTestUtils
class BaseEnvSuitTestSuite:
- """
- to prevent the full integrations tests that extend the partial integrations tests running both sets of tests,
- the test suite is nested in a base class, to keep it at a separate package-level
- https://stackoverflow.com/questions/1323455/python-unit-test-with-base-and-sub-class
- """
-
class EnvSuitTestSuite(unittest.TestCase):
TEST_OUT_PATH: str = None
diff --git a/tests/integration/test_suites/survey_test_suite.py b/tests/integration/test_suites/survey_test_suite.py
index 216a772..0ab19ee 100644
--- a/tests/integration/test_suites/survey_test_suite.py
+++ b/tests/integration/test_suites/survey_test_suite.py
@@ -7,12 +7,6 @@ from integration.partial.integration_test_utils import IntegrationTestUtils
class BaseSurveyTestSuite:
- """
- to prevent the full integrations tests that extend the partial integrations tests running both sets of tests,
- the test suite is nested in a base class, to keep it at a separate package-level
- https://stackoverflow.com/questions/1323455/python-unit-test-with-base-and-sub-class
- """
-
class SurveyTestSuite(unittest.TestCase):
TEST_OUT_PATH: str = None
--
GitLab