diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 53a52e6ac20145666f8fda61788dff4189250d65..eecd53784b59a3a3a0fa97fda19f685f178def25 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,18 +7,22 @@ variables:
OUTPUT_DIR: "$CI_PROJECT_DIR/outputs/"
TEST_WORKSPACE_DIR: "/storage/app/EWS_prod/regions/EastAfrica/test_workspace/"
- FULL_FAT_RUN_DATE:
- value: "default"
- options:
- - "default"
- - "today"
- - "yesterday"
- - "custom"
- description: "Full-fat tests will pull data from the deployment environment. For robustness, the default option is a
- pre-selected date we know works. Select \"today\" or \"yesterday\" to test with recent live data. If \"custom\" is
- selected, manually enter the key value pair using the format in this example: CUSTOM_RUN_DATE 20230115"
+# FULL_FAT_RUN_DATE_TYPE:
+# value: "default"
+# options:
+# - "default"
+# - "today"
+# - "yesterday"
+# - "custom"
+# description: "Full-fat tests will pull data from the deployment environment. For robustness, the default option is a
+# pre-selected date we know works. Select \"today\" or \"yesterday\" to test with recent live data. If \"custom\" is
+# selected, manually enter the key value pair using the format in this example: CUSTOM_RUN_DATE 20230115"
- CUSTOM_RUN_DATE: "NOT_SET"
+ CUSTOM_RUN_DATE:
+ value: "20231127"
+ description: "Full-fat tests will pull data from the deployment environment. For robustness, the default option is a
+ pre-selected date we know works. To override the default date, manually enter the key value pair using the format
+ in this example e.g.: CUSTOM_RUN_DATE 20230115"
cache:
paths:
@@ -42,7 +46,7 @@ stages:
- echo "output dir - " $OUTPUT_DIR
- echo "packages dir - " $PACKAGES_DIR
- echo "pip dir - " $PIP_CACHE_DIR
- - echo "full-fat run date - " $FULL_FAT_RUN_DATE
+ - echo "full-fat run date type - " FULL_FAT_RUN_DATE_TYPE
- echo "custom full-fat run date - " $CUSTOM_RUN_DATE
- source /storage/app/EWS_prod/envs/venv/ews_prod/bin/activate
@@ -120,7 +124,7 @@ full_fat_depo:
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--test_report_dir $CI_PROJECT_DIR/test_reports
- --run_date_type $FULL_FAT_RUN_DATE
+ --run_date_type $FULL_FAT_RUN_DATE_TYPE
--custom_run_date $CUSTOM_RUN_DATE
full_fat_env_suit:
@@ -137,7 +141,7 @@ full_fat_env_suit:
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/env_suit_config_EastAfrica_fc_live.json
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
- --run_date_type $FULL_FAT_RUN_DATE
+ --run_date_type $FULL_FAT_RUN_DATE_TYPE
--custom_run_date $CUSTOM_RUN_DATE
full_fat_survey:
@@ -154,7 +158,7 @@ full_fat_survey:
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/survey_config_EastAfrica_fc_live.json
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
- --run_date_type $FULL_FAT_RUN_DATE
+ --run_date_type $FULL_FAT_RUN_DATE_TYPE
--custom_run_date $CUSTOM_RUN_DATE
full_fat_advisory:
@@ -166,13 +170,14 @@ full_fat_advisory:
- >
cd $CI_PROJECT_DIR/tests/integration/full/
+ SURVEY_RUN_DATE=$(date -d "$CUSTOM_RUN_DATE -1 day" +%Y%m%d)
+
python3 full_test_survey.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/survey_config_EastAfrica_fc_live.json
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
- --run_date_type $FULL_FAT_RUN_DATE
- --custom_run_date $CUSTOM_RUN_DATE
+ --custom_run_date $SURVEY_RUN_DATE
--custom_dir_prefix "temp_advisory_2024-07-11_102855"
python3 full_test_deposition.py
@@ -181,7 +186,6 @@ full_fat_advisory:
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--test_report_dir $CI_PROJECT_DIR/test_reports
- --run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
--custom_dir_prefix "temp_advisory_2024-07-11_102855"
@@ -190,7 +194,6 @@ full_fat_advisory:
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/env_suit_config_EastAfrica_fc_live.json
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
- --run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
--custom_dir_prefix "temp_advisory_2024-07-11_102855"
@@ -199,7 +202,6 @@ full_fat_advisory:
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/advisory_config_EastAfrica_fc_live.json
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
- --run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
--custom_dir_prefix "temp_advisory_2024-07-11_102855"
@@ -217,7 +219,7 @@ full_fat_epi:
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/survey_config_EastAfrica_fc_live.json
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
- --run_date_type $FULL_FAT_RUN_DATE
+ --run_date_type $FULL_FAT_RUN_DATE_TYPE
--custom_run_date $CUSTOM_RUN_DATE
--custom_dir_prefix "temp_advisory_$RUN_TIME"
@@ -227,7 +229,7 @@ full_fat_epi:
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--test_report_dir $CI_PROJECT_DIR/test_reports
- --run_date_type $FULL_FAT_RUN_DATE
+ --run_date_type $FULL_FAT_RUN_DATE_TYPE
--custom_run_date $CUSTOM_RUN_DATE
--custom_dir_prefix "temp_advisory_$RUN_TIME"
@@ -236,7 +238,7 @@ full_fat_epi:
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/env_suit_config_EastAfrica_fc_live.json
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
- --run_date_type $FULL_FAT_RUN_DATE
+ --run_date_type $FULL_FAT_RUN_DATE_TYPE
--custom_run_date $CUSTOM_RUN_DATE
--custom_dir_prefix "temp_advisory_$RUN_TIME"
@@ -245,7 +247,7 @@ full_fat_epi:
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/epi_config_EastAfrica_fc_live.json
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
- --run_date_type $FULL_FAT_RUN_DATE
+ --run_date_type $FULL_FAT_RUN_DATE_TYPE
--custom_run_date $CUSTOM_RUN_DATE
--custom_dir_prefix "temp_advisory_$RUN_TIME"
diff --git a/tests/integration/full/full_test_advisory.py b/tests/integration/full/full_test_advisory.py
index 46bd3ad8f841144ee3b9c05c6b2dd4b3a654d45e..1e9e235ce8cd0be6c19752d2bcab7f716a4e899f 100644
--- a/tests/integration/full/full_test_advisory.py
+++ b/tests/integration/full/full_test_advisory.py
@@ -33,7 +33,7 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
# FullTestAdvisory.run_dependent_pipelines()
FullTestAdvisory.run_advisory_pipeline()
else:
- string = f"output in {IntegrationTestUtils.TEST_OUT_PATH} already written, skipping rerun"
+ string = f"output in {IntegrationTestUtils.TEST_JOB_DIR} already written, skipping rerun"
print(string)
i = 0
diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py
index c1d00db19463a9a0f136bca9de039a9e7cedb0f1..21688d5944b6265a96fd65460923f1945a369041 100644
--- a/tests/integration/partial/integration_test_utils.py
+++ b/tests/integration/partial/integration_test_utils.py
@@ -62,7 +62,7 @@ class IntegrationTestUtils:
parser.add_argument('--outdir', required = True)
parser.add_argument('--email_cred', required = True)
parser.add_argument('--test_report_dir', required = False)
- parser.add_argument('--run_date_type', required = False)
+ # parser.add_argument('--run_date_type', required = False)
parser.add_argument('--custom_run_date', required = False)
parser.add_argument('--custom_dir_prefix', required = False, help = "prefix for the test dir, a sensible value will be set as default")
parser.add_argument('unittest_args', nargs='*')
@@ -92,7 +92,7 @@ class IntegrationTestUtils:
_outdir: str = _args.outdir
_email_cred_path: str = _args.email_cred
_test_report_dir: str = _args.test_report_dir
- _run_date_type: str = _args.run_date_type
+ # _run_date_type: str = _args.run_date_type
_custom_run_date: str = _args.custom_run_date
_custom_dir_prefix: str = _args.custom_dir_prefix
@@ -114,8 +114,7 @@ class IntegrationTestUtils:
IntegrationTestUtils.TEST_OUT_PATH = _outdir + prefix + os.sep
IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path
- IntegrationTestUtils.TEST_START_DATE = IntegrationTestUtils.generate_run_date(_run_date_type,
- _custom_run_date)
+ IntegrationTestUtils.TEST_START_DATE = _custom_run_date
IntegrationTestUtils.TEST_JOB_DIR = os.path.join(IntegrationTestUtils.TEST_OUT_PATH,
f"{processor_dir}_" +
@@ -285,25 +284,25 @@ class IntegrationTestUtils:
return yesterday_string
- @staticmethod
- def generate_run_date(run_date_type: str,
- custom_run_date: str) -> str:
-
- if run_date_type == "today":
- print("today")
- today = datetime.date.today()
- today_string = today.strftime("%Y%m%d")
- result = today_string
- elif run_date_type == "yesterday":
- print("yesterday")
- today = datetime.date.today()
- yesterday: datetime = today - datetime.timedelta(days = 1)
- yesterday_string = yesterday.strftime("%Y%m%d")
- result = yesterday_string
- elif run_date_type == "custom":
- result = custom_run_date
- else:
- print("default")
- result = "20231127"
-
- return result
+ # @staticmethod
+ # def generate_run_date(run_date_type: str,
+ # custom_run_date: str) -> str:
+ #
+ # if run_date_type == "today":
+ # print("today")
+ # today = datetime.date.today()
+ # today_string = today.strftime("%Y%m%d")
+ # result = today_string
+ # elif run_date_type == "yesterday":
+ # print("yesterday")
+ # today = datetime.date.today()
+ # yesterday: datetime = today - datetime.timedelta(days = 1)
+ # yesterday_string = yesterday.strftime("%Y%m%d")
+ # result = yesterday_string
+ # elif run_date_type == "custom":
+ # result = custom_run_date
+ # else:
+ # print("default")
+ # result = "20231127"
+ #
+ # return result