From 0366e5db4e54b598c52af923015849709a3f87bd Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Thu, 11 Jul 2024 10:50:00 +0100
Subject: [PATCH] more work to make full-fat tests wotk with new resources
structure
---
.gitlab-ci.yml | 12 ++++
tests/integration/full/full_test_advisory.py | 66 ++++++++++++-------
.../integration/full/full_test_deposition.py | 11 ++--
tests/integration/full/full_test_env_suit.py | 10 +--
tests/integration/full/full_test_epi.py | 16 ++---
tests/integration/full/full_test_survey.py | 10 +--
.../partial/integration_test_utils.py | 12 +++-
7 files changed, 85 insertions(+), 52 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ae80f1c..f5ded70 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -35,6 +35,8 @@ stages:
- BRANCH="main"
- echo "branch - " $BRANCH
+ - RUN_TIME=$(date '+%Y%m%d_%H%M%S')
+ - echo "run time - " $run_time
- PACKAGES_DIR="$CI_PROJECT_DIR/packages/"
- echo "project dir - " $CI_PROJECT_DIR
- echo "output dir - " $OUTPUT_DIR
@@ -164,6 +166,16 @@ full_fat_advisory:
- >
cd $CI_PROJECT_DIR/tests/integration/full/
+# RUN SURVEY
+ python3 full_test_survey.py
+ --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
+ --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/survey_config_EastAfrica_fc_live.json
+ --outdir ${TEST_WORKSPACE_DIR}/integration/full/
+ --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
+ --run_date_type $FULL_FAT_RUN_DATE
+ --custom_run_date $CUSTOM_RUN_DATE
+ --custom_dir_prefix "ADVISORY_$RUN_TIME"
+
python3 full_test_advisory.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/advisory_EastAfrica_fc_live.json
diff --git a/tests/integration/full/full_test_advisory.py b/tests/integration/full/full_test_advisory.py
index a46ca90..46bd3ad 100644
--- a/tests/integration/full/full_test_advisory.py
+++ b/tests/integration/full/full_test_advisory.py
@@ -6,6 +6,7 @@ from ews.coordinator.processor_advisory import ProcessorAdvisory
from ews.coordinator.processor_deposition import ProcessorDeposition
from ews.coordinator.processor_environment import ProcessorEnvironment
from ews.coordinator.processor_surveys import ProcessorSurveys
+from ews.coordinator.utils.processor_utils import parse_json_file_with_tokens
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.advisory_test_suite import BaseAdvisoryTestSuite
from integration.test_suites.depo_test_suite import BaseDepoTestSuite
@@ -25,11 +26,11 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
self.set_expected_values()
- if IntegrationTestUtils.TEST_OUT_PATH is None \
- or not os.listdir(IntegrationTestUtils.TEST_OUT_PATH):
- # if True:
+ if IntegrationTestUtils.TEST_JOB_DIR is None \
+ or not os.path.exists(IntegrationTestUtils.TEST_JOB_DIR):
+ # if True:
FullTestAdvisory.write_temp_run_config_files()
- FullTestAdvisory.run_dependent_pipelines()
+ # FullTestAdvisory.run_dependent_pipelines()
FullTestAdvisory.run_advisory_pipeline()
else:
string = f"output in {IntegrationTestUtils.TEST_OUT_PATH} already written, skipping rerun"
@@ -39,28 +40,45 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
@staticmethod
def write_temp_run_config_files():
- default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
- default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
- run_dict: dict = copy.deepcopy(default_config_dict)
- run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
- # we need to run the env suit pipeline as well
- run_dict['Environment']['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['Environment']['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['Environment']['OUTPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
-
- #make sure we activate server download
- run_dict['Survey']['SkipServerDownload'] = False
-
- # survey_cred_file: str = run_dict['Survey']['ServerCredentialsFile']
- # run_dict['Survey']['ServerCredentialsFile'] = survey_cred_file.replace("Cred-ODK-EIAR.json", "Cred-ODK-EIAR-NO-WRSIS.json")
- run_dict['Survey']['FormEdits'].pop('WRSIS', None)
-
- # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
+
+ # default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
+ # default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
+ # run_dict: dict = copy.deepcopy(default_config_dict)
+ # run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
+ # run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
+ #
+ # # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
+ # os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
+ # IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ # IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)
+
+ """
+ IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH will have been set from the command line args, and point to the
+ production sys config file. We need to modify this file to point to the test output directory.
+ :return:
+ """
+ sys_config = IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH
+ sys_config_dict: dict = parse_json_file_with_tokens(sys_config)
+
+ sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
+ sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
+
os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)
+ job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_advisory_config.json"
+ IntegrationTestUtils.write_json_file(sys_config_dict, job_sys_config_file)
+
+ # now set the sys config file path to the new job-specific file
+ IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = job_sys_config_file
+
+ run_config = IntegrationTestUtils.RUN_CONFIG_FILE_PATH
+ run_config_temp_dict: dict = parse_json_file_with_tokens(run_config, sys_config_dict)
+ run_dict: dict = copy.deepcopy(run_config_temp_dict)
+
+ job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_advisory_config.json"
+ IntegrationTestUtils.write_json_file(run_dict, job_run_config_file)
+ # set the run config file path to the new job-specific file
+ IntegrationTestUtils.RUN_CONFIG_FILE_PATH = job_run_config_file
@staticmethod
def run_dependent_pipelines():
diff --git a/tests/integration/full/full_test_deposition.py b/tests/integration/full/full_test_deposition.py
index db55291..5f72097 100644
--- a/tests/integration/full/full_test_deposition.py
+++ b/tests/integration/full/full_test_deposition.py
@@ -21,14 +21,13 @@ class FullTestDeposition(BaseDepoTestSuite.DepoTestSuite):
self.set_expected_values()
- path = IntegrationTestUtils.TEST_OUT_PATH
- if path is None \
- or not os.listdir(IntegrationTestUtils.TEST_OUT_PATH):
+ if IntegrationTestUtils.TEST_JOB_DIR is None \
+ or not os.path.exists(IntegrationTestUtils.TEST_JOB_DIR):
# if True:
FullTestDeposition.write_temp_run_config_file()
FullTestDeposition.run_depo_pipeline()
else:
- print(f"output in {IntegrationTestUtils.TEST_OUT_PATH} already written, skipping rerun")
+ print(f"output in {IntegrationTestUtils.TEST_JOB_DIR} already written, skipping rerun")
@staticmethod
@@ -46,7 +45,7 @@ class FullTestDeposition(BaseDepoTestSuite.DepoTestSuite):
sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+ job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_deposition_config.json"
IntegrationTestUtils.write_json_file(sys_config_dict, job_sys_config_file)
# now set the sys config file path to the new job-specific file
@@ -56,7 +55,7 @@ class FullTestDeposition(BaseDepoTestSuite.DepoTestSuite):
run_config_temp_dict: dict = parse_json_file_with_tokens(run_config, sys_config_dict)
run_dict: dict = copy.deepcopy(run_config_temp_dict)
- job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_deposition_config.json"
IntegrationTestUtils.write_json_file(run_dict, job_run_config_file)
# set the run config file path to the new job-specific file
diff --git a/tests/integration/full/full_test_env_suit.py b/tests/integration/full/full_test_env_suit.py
index a8537e3..c4c4f17 100644
--- a/tests/integration/full/full_test_env_suit.py
+++ b/tests/integration/full/full_test_env_suit.py
@@ -20,12 +20,12 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
self.set_expected_values()
- if IntegrationTestUtils.TEST_OUT_PATH is None \
- or not os.listdir(IntegrationTestUtils.TEST_OUT_PATH):
+ if IntegrationTestUtils.TEST_JOB_DIR is None \
+ or not os.path.exists(IntegrationTestUtils.TEST_JOB_DIR):
FullTestEnvSuit.write_temp_run_config_file()
FullTestEnvSuit.run_env_pipeline()
else:
- print(f"output in {IntegrationTestUtils.TEST_OUT_PATH} already written, skipping rerun")
+ print(f"output in {IntegrationTestUtils.TEST_JOB_DIR} already written, skipping rerun")
@staticmethod
@@ -42,7 +42,7 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+ job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_advisory_config.json"
IntegrationTestUtils.write_json_file(sys_config_dict, job_sys_config_file)
# now set the sys config file path to the new job-specific file
@@ -57,7 +57,7 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
run_dict['EWS-Plotting']['Ethiopia']['FilterForCountry'] = "True"
run_dict['EWS-Plotting']['Kenya']['FilterForCountry'] = "True"
- job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_deposition_config.json"
IntegrationTestUtils.write_json_file(run_dict, job_run_config_file)
# set the run config file path to the new job-specific file
diff --git a/tests/integration/full/full_test_epi.py b/tests/integration/full/full_test_epi.py
index a4a8faa..792df11 100644
--- a/tests/integration/full/full_test_epi.py
+++ b/tests/integration/full/full_test_epi.py
@@ -23,14 +23,14 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
self.set_expected_values()
- if IntegrationTestUtils.TEST_OUT_PATH is None \
- or not os.listdir(IntegrationTestUtils.TEST_OUT_PATH):
- # if True:
+ if IntegrationTestUtils.TEST_JOB_DIR is None \
+ or not os.path.exists(IntegrationTestUtils.TEST_JOB_DIR):
+ # if True:
FullTestEpi.write_temp_run_config_files()
- FullTestEpi.run_dependent_pipelines()
+ # FullTestEpi.run_dependent_pipelines()
FullTestEpi.run_epi_pipeline()
else:
- print(f"output in {IntegrationTestUtils.TEST_OUT_PATH} already written, skipping rerun")
+ print(f"output in {IntegrationTestUtils.TEST_JOB_DIR} already written, skipping rerun")
@staticmethod
@@ -57,7 +57,7 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+ job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_epi_config.json"
IntegrationTestUtils.write_json_file(sys_config_dict, job_sys_config_file)
# now set the sys config file path to the new job-specific file
@@ -70,8 +70,6 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
run_dict['Environment']['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
run_dict['Environment']['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
run_dict['Environment']['OUTPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
- # make sure we activate server download
- run_dict['Survey']['SkipServerDownload'] = False
# change the config for the Epi so it only runs on a single past date. Requires use of a different
# FileListerFunction and a different calculation span
@@ -89,7 +87,7 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
previous_day_string: str = IntegrationTestUtils.get_day_before_as_string(IntegrationTestUtils.TEST_START_DATE)
run_dict['SeasonStartString'] = previous_day_string
- job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_epi_config.json"
IntegrationTestUtils.write_json_file(run_dict, job_run_config_file)
# set the run config file path to the new job-specific file
diff --git a/tests/integration/full/full_test_survey.py b/tests/integration/full/full_test_survey.py
index fdb71f5..831c4cb 100644
--- a/tests/integration/full/full_test_survey.py
+++ b/tests/integration/full/full_test_survey.py
@@ -19,12 +19,12 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
self.set_expected_values()
- if IntegrationTestUtils.TEST_OUT_PATH is None \
- or not os.listdir(IntegrationTestUtils.TEST_OUT_PATH):
+ if IntegrationTestUtils.TEST_JOB_DIR is None \
+ or not os.path.exists(IntegrationTestUtils.TEST_JOB_DIR):
FullTestSurvey.write_temp_run_config_file()
FullTestSurvey.run_survey_pipeline()
else:
- print(f"output in {IntegrationTestUtils.TEST_OUT_PATH} already written, skipping rerun")
+ print(f"output in {IntegrationTestUtils.TEST_JOB_DIR} already written, skipping rerun")
@staticmethod
@@ -41,7 +41,7 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+ job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_survey_config.json"
IntegrationTestUtils.write_json_file(sys_config_dict, job_sys_config_file)
# now set the sys config file path to the new job-specific file
@@ -52,7 +52,7 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
run_dict: dict = copy.deepcopy(run_config_temp_dict)
run_dict['SkipServerDownload'] = False
- job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ job_run_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_survey_config.json"
IntegrationTestUtils.write_json_file(run_dict, job_run_config_file)
# set the run config file path to the new job-specific file
diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py
index ebc59a0..48cadf3 100644
--- a/tests/integration/partial/integration_test_utils.py
+++ b/tests/integration/partial/integration_test_utils.py
@@ -64,6 +64,7 @@ class IntegrationTestUtils:
parser.add_argument('--test_report_dir', required = False)
parser.add_argument('--run_date_type', required = False)
parser.add_argument('--custom_run_date', required = False)
+ parser.add_argument('--custom_dir_prefix', required = False, help = "prefix for the test dir, a sensible value will be set as default")
parser.add_argument('unittest_args', nargs='*')
return parser
@@ -93,6 +94,7 @@ class IntegrationTestUtils:
_test_report_dir: str = _args.test_report_dir
_run_date_type: str = _args.run_date_type
_custom_run_date: str = _args.custom_run_date
+ _custom_dir_prefix: str = _args.custom_dir_prefix
"""
We store paths to the production run json files here, these are then read into a dict, output paths are
@@ -102,9 +104,13 @@ class IntegrationTestUtils:
IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = _sys_config_file
IntegrationTestUtils.RUN_CONFIG_FILE_PATH = _run_config_file
- nowstring: str = IntegrationTestUtils.get_now_string()
- prefix: str = f"temp_{test_prefix}_" + nowstring
- # prefix: str = f"temp_{test_prefix}"
+ if _custom_dir_prefix is None:
+ nowstring: str = IntegrationTestUtils.get_now_string()
+ prefix: str = f"temp_{test_prefix}_" + nowstring
+ # prefix: str = f"temp_{test_prefix}"
+ else:
+ prefix = _custom_dir_prefix
+
IntegrationTestUtils.TEST_OUT_PATH = _outdir + prefix + os.sep
IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path
--
GitLab