diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 28e030785b97b3af53ac5276b5150ea32804d7a0..05395fa450a35b6f599f112d53e2aa794814c811 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -92,7 +92,7 @@ stages: # - echo $PYTHONPATH half_fat_tests: - when: always + when: manual stage: half-fat extends: .half_fat_before_script image: lb584/ews_coordinator:latest @@ -162,3 +162,36 @@ full_fat_depo: - $OUTPUT_DIR/*/log.txt - $OUTPUT_DIR/*.txt expire_in: 10 days + +full_fat_survey: + when: manual + stage: full-fat + extends: .half_fat_before_script + image: lb584/ews_coordinator:latest + tags: ["ewas-production"] + script: + - > + cd $CI_PROJECT_DIR/tests/integration/full/ + + python3 full_test_survey.py + --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json + --outdir $OUTPUT_DIR + --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json + --run_date_type $FULL_FAT_RUN_DATE + --custom_run_date $CUSTOM_RUN_DATE + + after_script: + - > + if [[ "$FULL_FAT_COPY_OUTPUT" == "true" ]]; + then + date=$(date '+%Y-%m-%d-%H%M'); + mkdir ${TEST_WORKSPACE_DIR}/integration/full/${date}; + cp -r $OUTPUT_DIR/* ${TEST_WORKSPACE_DIR}/integration/full/${date}; + fi + + artifacts: + when: on_failure + paths: + - $OUTPUT_DIR/*/log.txt + - $OUTPUT_DIR/*.txt + expire_in: 10 days diff --git a/tests/integration/full/full_test_deposition.py b/tests/integration/full/full_test_deposition.py index 5973a63bdbb8c994af46c40fb897845a264ed0c6..b9c20ba5cd8106b99dee01e274d9835c40cbe683 100644 --- a/tests/integration/full/full_test_deposition.py +++ b/tests/integration/full/full_test_deposition.py @@ -1,6 +1,5 @@ import argparse import copy -import datetime import glob import os import sys @@ -9,17 +8,17 @@ import unittest from integration.partial.integration_test_utils import IntegrationTestUtils -class TestFullDeposition(unittest.TestCase): +class FullTestDeposition(unittest.TestCase): - TEST_OUT_PATH: str = "not_set" + TEST_OUT_PATH: str = "NOT_SET" TEST_START_DATE: str = 'NOT_SET' - TEST_JOB_DIR: str = "DEPOSITION_" + TEST_START_DATE + TEST_JOB_DIR: str = "NOT_SET" @classmethod def setUpClass(cls) -> None: - TestFullDeposition.write_temp_run_config_file() - TestFullDeposition.run_depo_pipeline() + FullTestDeposition.write_temp_run_config_file() + FullTestDeposition.run_depo_pipeline() pass @@ -28,10 +27,10 @@ class TestFullDeposition(unittest.TestCase): default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config) run_dict: dict = copy.deepcopy(default_config_dict) - run_dict['WorkspacePathout'] = TestFullDeposition.TEST_OUT_PATH - run_dict['WorkspacePath'] = TestFullDeposition.TEST_OUT_PATH + run_dict['WorkspacePathout'] = FullTestDeposition.TEST_OUT_PATH + run_dict['WorkspacePath'] = FullTestDeposition.TEST_OUT_PATH - IntegrationTestUtils.TEMP_CONFIG_FILE_PATH = TestFullDeposition.TEST_OUT_PATH + "temp_config.json" + IntegrationTestUtils.TEMP_CONFIG_FILE_PATH = FullTestDeposition.TEST_OUT_PATH + "temp_config.json" IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH) @@ -39,12 +38,12 @@ class TestFullDeposition(unittest.TestCase): def run_depo_pipeline(): component = 'Deposition' IntegrationTestUtils.run_external_pipeline(component, - TestFullDeposition.TEST_START_DATE, + FullTestDeposition.TEST_START_DATE, IntegrationTestUtils.EMAIL_CRED_PATH) def test_standard_run_input_status_success(self): - status_file_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR, "STATUS_SUCCESS") + status_file_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR, "STATUS_SUCCESS") success_file_exists: bool = os.path.isfile(status_file_path) self.assertTrue(success_file_exists) @@ -55,10 +54,10 @@ class TestFullDeposition(unittest.TestCase): (at least past the region iteration) """ - east_africa_image_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR, + east_africa_image_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR, "plotting", "eastafrica", "images", "Weekly", "deposition_eastafrica_leaf_rust_total_*_map.png") - ethiopia_image_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR, + ethiopia_image_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR, "plotting", "ethiopia", "images", "Weekly", "deposition_ethiopia_leaf_rust_total_*_map.png") @@ -69,9 +68,9 @@ class TestFullDeposition(unittest.TestCase): def test_standard_run_all_input_csvs_produced(self): - east_africa_csv_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR, + east_africa_csv_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR, "plotting", "eastafrica", "input_csvs", "*.csv") - ethiopia_csv_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR, + ethiopia_csv_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR, "plotting", "ethiopia", "input_csvs", "*.csv") ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_csv_path) @@ -81,9 +80,9 @@ class TestFullDeposition(unittest.TestCase): def test_standard_run_all_images_produced(self): - east_africa_image_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR, + east_africa_image_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR, "plotting", "eastafrica", "images", "Weekly", "*.png") - ethiopia_image_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR, + ethiopia_image_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR, "plotting", "ethiopia", "images", "Weekly", "*.png") ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_image_path) @@ -110,32 +109,12 @@ if __name__ == '__main__': IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH = _config_file - TestFullDeposition.TEST_OUT_PATH = _outdir + FullTestDeposition.TEST_OUT_PATH = _outdir IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path - if _run_date_type == "today": - print("today") - today = datetime.date.today() - today_string = today.strftime("%Y%m%d") - TestFullDeposition.TEST_START_DATE = today_string - TestFullDeposition.TEST_JOB_DIR = "DEPOSITION_" + TestFullDeposition.TEST_START_DATE - print(TestFullDeposition.TEST_JOB_DIR) - elif _run_date_type == "yesterday": - print("yesterday") - today = datetime.date.today() - yesterday: datetime = today - datetime.timedelta(days=1) - yesterday_string = yesterday.strftime("%Y%m%d") - TestFullDeposition.TEST_START_DATE = yesterday_string - TestFullDeposition.TEST_JOB_DIR = "DEPOSITION_" + TestFullDeposition.TEST_START_DATE - print(TestFullDeposition.TEST_JOB_DIR) - elif _run_date_type == "custom": - print(_custom_run_date) - TestFullDeposition.TEST_START_DATE = _custom_run_date - TestFullDeposition.TEST_JOB_DIR = "DEPOSITION_" + _custom_run_date - else: - print("default") - TestFullDeposition.TEST_START_DATE = "20230126" - TestFullDeposition.TEST_JOB_DIR = "DEPOSITION_" + TestFullDeposition.TEST_START_DATE + FullTestDeposition.TEST_START_DATE = IntegrationTestUtils.generate_run_date(_run_date_type, _custom_run_date) + + FullTestDeposition.TEST_JOB_DIR = "DEPOSITION_" + FullTestDeposition.TEST_START_DATE # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone) sys.argv[1:] = _args.unittest_args diff --git a/tests/integration/full/full_test_survey.py b/tests/integration/full/full_test_survey.py index d9b2c167dc794a6fa07636f2f8662f6a00ba2365..a90ad1b8664239f4bd71312c72a492e39b86b3c5 100644 --- a/tests/integration/full/full_test_survey.py +++ b/tests/integration/full/full_test_survey.py @@ -9,9 +9,9 @@ from integration.partial.integration_test_utils import IntegrationTestUtils class FullTestSurvey(unittest.TestCase): - TEST_OUT_PATH: str = "not_set" - TEST_START_DATE: str = '20230126' - TEST_JOB_DIR: str = "SURVEYDATA_" + TEST_START_DATE + TEST_OUT_PATH: str = "NOT_SET" + TEST_START_DATE: str = 'NOT_SET' + TEST_JOB_DIR: str = "NOT_SET" @classmethod def setUpClass(cls) -> None: @@ -33,9 +33,9 @@ class FullTestSurvey(unittest.TestCase): IntegrationTestUtils.TEMP_CONFIG_FILE_PATH = FullTestSurvey.TEST_OUT_PATH + "temp_config.json" IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH) - @staticmethod - def unpack_dependencies(): - IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_SURVEY_FILE_PATH, FullTestSurvey.TEST_OUT_PATH) + # @staticmethod + # def unpack_dependencies(): + # IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_SURVEY_FILE_PATH, FullTestSurvey.TEST_OUT_PATH) @staticmethod def run_survey_pipeline(): @@ -64,18 +64,24 @@ if __name__ == '__main__': parser.add_argument('--config', required = True) parser.add_argument('--outdir', required = True) parser.add_argument('--email_cred', required = True) + parser.add_argument('--run_date_type', required = False) + parser.add_argument('--custom_run_date', required = False) parser.add_argument('unittest_args', nargs='*') _args = parser.parse_args() _config_file: str = _args.config _outdir: str = _args.outdir _email_cred_path: str = _args.email_cred + _run_date_type: str = _args.run_date_type + _custom_run_date: str = _args.custom_run_date IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH = _config_file FullTestSurvey.TEST_OUT_PATH = _outdir IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path + FullTestSurvey.TEST_START_DATE = IntegrationTestUtils.generate_run_date(_run_date_type, _custom_run_date) + # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone) sys.argv[1:] = _args.unittest_args unittest.main() diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py index 943f0d1dab58e4e0b408bbf41ce5ef1789944ece..1b8ee79e8362b33f0065cf2d452e8162bf35775b 100644 --- a/tests/integration/partial/integration_test_utils.py +++ b/tests/integration/partial/integration_test_utils.py @@ -1,7 +1,7 @@ +import datetime import glob import json import os -from datetime import datetime from importlib import reload from typing import List from zipfile import ZipFile @@ -31,7 +31,7 @@ class IntegrationTestUtils: @staticmethod def get_now_string() -> str: - nowstring: str = datetime.today().strftime('%Y-%m-%d_%H%M%S') + nowstring: str = datetime.date.today().strftime('%Y-%m-%d_%H%M%S') return nowstring @@ -141,3 +141,27 @@ class IntegrationTestUtils: except SystemExit: # we will eventually want to throw these to the calling class to be dealt with pass + + + @staticmethod + def generate_run_date(run_date_type: str, + custom_run_date: str) -> str: + + if run_date_type == "today": + print("today") + today = datetime.date.today() + today_string = today.strftime("%Y%m%d") + result = today_string + elif run_date_type == "yesterday": + print("yesterday") + today = datetime.date.today() + yesterday: datetime = today - datetime.timedelta(days = 1) + yesterday_string = yesterday.strftime("%Y%m%d") + result = yesterday_string + elif run_date_type == "custom": + result = custom_run_date + else: + print("default") + result = "20230126" + + return result