diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36c0e42954cdce1c1bc5eb3fbca42aa87c226394..6b683b313eee4ababca0a2fb712166aac39e28b1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,7 +70,7 @@ stages: - ls $PACKAGES_DIR half_fat_tests: - when: always + when: manual stage: half-fat extends: .before_script @@ -113,7 +113,8 @@ full_fat_depo: cd $CI_PROJECT_DIR/tests/integration/full/ python3 full_test_deposition.py - --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json + --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json + --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/depo_config_EastAfrica_fc_live.json --outdir ${TEST_WORKSPACE_DIR}/integration/full/ --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json --test_report_dir $CI_PROJECT_DIR/test_reports @@ -130,7 +131,8 @@ full_fat_env_suit: cd $CI_PROJECT_DIR/tests/integration/full/ python3 full_test_env_suit.py - --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json + --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json + --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/env_suit_config_EastAfrica_fc_live.json --outdir ${TEST_WORKSPACE_DIR}/integration/full/ --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json --run_date_type $FULL_FAT_RUN_DATE @@ -146,7 +148,8 @@ full_fat_survey: cd $CI_PROJECT_DIR/tests/integration/full/ python3 full_test_survey.py - --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json + --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json + --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/survey_EastAfrica_fc_live.json --outdir ${TEST_WORKSPACE_DIR}/integration/full/ --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json --run_date_type $FULL_FAT_RUN_DATE @@ -162,7 +165,8 @@ full_fat_advisory: cd $CI_PROJECT_DIR/tests/integration/full/ python3 full_test_advisory.py - --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json + --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json + --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/advisory_EastAfrica_fc_live.json --outdir ${TEST_WORKSPACE_DIR}/integration/full/ --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json --run_date_type $FULL_FAT_RUN_DATE @@ -178,7 +182,8 @@ full_fat_epi: cd $CI_PROJECT_DIR/tests/integration/full/ python3 full_test_epi.py - --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json + --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json + --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/epi_config_EastAfrica_fc_live.json --outdir ${TEST_WORKSPACE_DIR}/integration/full/ --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json --run_date_type $FULL_FAT_RUN_DATE diff --git a/configs/docker/run/launchAndRunDockerEWS.sh b/configs/docker/run/launchAndRunDockerEWS.sh index 96565aacd6e30bffd0286415362afb2d307e3fb9..9cca8b74fd81eecc090d2f2cd4b8be8668783431 100755 --- a/configs/docker/run/launchAndRunDockerEWS.sh +++ b/configs/docker/run/launchAndRunDockerEWS.sh @@ -11,8 +11,8 @@ run_config='not_set' rundate=$(date '+%Y%m%d') # default today -SHORT=p:c:s:l:h -LONG=component:,config:,rundate:,noupload::,islive::,clearup::,loglevel::,help +SHORT=p:s:c:d:l:h +LONG=component:,sys_config:,run_config:,rundate:,noupload::,islive::,clearup::,loglevel::,help OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") echo $OPTS eval set -- "$OPTS" @@ -24,7 +24,7 @@ do component="$2" shift 2 ;; - -sc | --sys_config ) + -s | --sys_config ) sys_config="$2" shift 2 ;; @@ -32,7 +32,7 @@ do run_config="$2" shift 2 ;; - -s | --rundate ) + -d | --rundate ) rundate="$2" shift 2 ;; @@ -87,7 +87,7 @@ else fi function run_coordinator() { - printf "component: %s\nconfig: %s\nrundate: %s\n" "$component" "$sys_config" "$run_config" "$rundate" + printf "component: %s\nsys_config: %s\nrun_config: %s\nrundate: %s\n" "$component" "$sys_config" "$run_config" "$rundate" printf "optional args:\n%s\n" "$upload $islive $clearup" config_base=$(basename $run_config) printf "%s\n" $config_base @@ -105,7 +105,7 @@ function run_coordinator() { "$component_script" \ -sc "$sys_config" \ -c "$run_config" \ - -s "$rundate" \ + -d "$rundate" \ -l "$loglevel" \ $upload \ $islive \ diff --git a/tests/integration/full/full_test_env_suit.py b/tests/integration/full/full_test_env_suit.py index 2c3e8295eac541fed1c393973afbf795ce49ab1d..ba9c0e73833efdcc2427ad6cb9e3341753886271 100644 --- a/tests/integration/full/full_test_env_suit.py +++ b/tests/integration/full/full_test_env_suit.py @@ -3,6 +3,7 @@ import os import sys from ews.coordinator.processor_environment import ProcessorEnvironment +from ews.coordinator.utils.processor_utils import parse_json_file_with_tokens from integration.partial.integration_test_utils import IntegrationTestUtils from integration.test_suites.env_suit_test_suite import BaseEnvSuitTestSuite @@ -30,24 +31,27 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite): @staticmethod def write_temp_run_config_file(): - sys_config_path = IntegrationTestUtils.DEFAULT_SYS_CONFIG_FILE_PATH - sys_config_dict: dict = IntegrationTestUtils.load_json_file(sys_config_path) + sys_config = IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH + sys_config_dict: dict = parse_json_file_with_tokens(sys_config) sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH - IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" - IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH) - default_config = IntegrationTestUtils.DEFAULT_ENV_SUIT_CONFIG_FILE_PATH - default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config) - run_dict: dict = copy.deepcopy(default_config_dict) + os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True) + job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json" + IntegrationTestUtils.write_json_file(sys_config_dict, job_sys_config_file) + + # now set the sys config file path to the new job-specific file + IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = job_sys_config_file + + run_config = IntegrationTestUtils.RUN_CONFIG_FILE_PATH + run_config_temp_dict: dict = IntegrationTestUtils.load_json_file(run_config) + run_dict: dict = copy.deepcopy(run_config_temp_dict) run_dict['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH run_dict['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH run_dict['OUTPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH run_dict['EWS-Plotting']['Ethiopia']['FilterForCountry'] = "True" run_dict['EWS-Plotting']['Kenya']['FilterForCountry'] = "True" - os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True) - IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json" IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)