Merge branch 'main' of gitlab_uis:gilligan-epid/wheat-rusts/ews-coordinator

.gitlab-ci.yml

@@ -70,7 +70,7 @@ stages:
       - ls $PACKAGES_DIR
 
 half_fat_tests:
-  when: always
+  when: manual
   stage: half-fat
   extends: .before_script
 
@@ -113,7 +113,8 @@ full_fat_depo:
       cd $CI_PROJECT_DIR/tests/integration/full/
       
       python3 full_test_deposition.py 
-      --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
+      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/depo_config_EastAfrica_fc_live.json
       --outdir ${TEST_WORKSPACE_DIR}/integration/full/
       --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
       --test_report_dir $CI_PROJECT_DIR/test_reports
@@ -130,7 +131,8 @@ full_fat_env_suit:
       cd $CI_PROJECT_DIR/tests/integration/full/
 
       python3 full_test_env_suit.py 
-      --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
+      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/env_suit_config_EastAfrica_fc_live.json
       --outdir ${TEST_WORKSPACE_DIR}/integration/full/
       --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
       --run_date_type $FULL_FAT_RUN_DATE
@@ -146,7 +148,8 @@ full_fat_survey:
       cd $CI_PROJECT_DIR/tests/integration/full/
       
       python3 full_test_survey.py 
-      --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
+      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/survey_EastAfrica_fc_live.json
       --outdir ${TEST_WORKSPACE_DIR}/integration/full/
       --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
       --run_date_type $FULL_FAT_RUN_DATE
@@ -162,7 +165,8 @@ full_fat_advisory:
       cd $CI_PROJECT_DIR/tests/integration/full/
 
       python3 full_test_advisory.py 
-      --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
+      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/advisory_EastAfrica_fc_live.json
       --outdir ${TEST_WORKSPACE_DIR}/integration/full/
       --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
       --run_date_type $FULL_FAT_RUN_DATE
@@ -178,7 +182,8 @@ full_fat_epi:
       cd $CI_PROJECT_DIR/tests/integration/full/
 
       python3 full_test_epi.py 
-      --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
+      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/epi_config_EastAfrica_fc_live.json
       --outdir ${TEST_WORKSPACE_DIR}/integration/full/
       --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
       --run_date_type $FULL_FAT_RUN_DATE

configs/docker/run/launchAndRunDockerEWS.sh

@@ -11,8 +11,8 @@ run_config='not_set'
 rundate=$(date '+%Y%m%d') # default today
 
 
-SHORT=p:c:s:l:h
-LONG=component:,config:,rundate:,noupload::,islive::,clearup::,loglevel::,help
+SHORT=p:s:c:d:l:h
+LONG=component:,sys_config:,run_config:,rundate:,noupload::,islive::,clearup::,loglevel::,help
 OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@")
 echo $OPTS
 eval set -- "$OPTS"
@@ -24,7 +24,7 @@ do
       component="$2"
       shift 2
       ;;
-    -sc | --sys_config )
+    -s | --sys_config )
       sys_config="$2"
       shift 2
       ;;
@@ -32,7 +32,7 @@ do
       run_config="$2"
       shift 2
       ;;
-    -s | --rundate )
+    -d | --rundate )
       rundate="$2"
       shift 2
       ;;
@@ -87,7 +87,7 @@ else
 fi
 
 function run_coordinator() {
-    printf "component: %s\nconfig: %s\nrundate: %s\n" "$component" "$sys_config" "$run_config" "$rundate"
+    printf "component: %s\nsys_config: %s\nrun_config: %s\nrundate: %s\n" "$component" "$sys_config" "$run_config" "$rundate"
     printf "optional args:\n%s\n" "$upload $islive $clearup"
     config_base=$(basename $run_config)
     printf "%s\n" $config_base
@@ -105,7 +105,7 @@ function run_coordinator() {
     "$component_script" \
     -sc "$sys_config" \
     -c "$run_config" \
-    -s "$rundate" \
+    -d "$rundate" \
     -l "$loglevel" \
     $upload \
     $islive \

tests/integration/full/full_test_env_suit.py

@@ -3,6 +3,7 @@ import os
 import sys
 
 from ews.coordinator.processor_environment import ProcessorEnvironment
+from ews.coordinator.utils.processor_utils import parse_json_file_with_tokens
 from integration.partial.integration_test_utils import IntegrationTestUtils
 from integration.test_suites.env_suit_test_suite import BaseEnvSuitTestSuite
 
@@ -30,24 +31,27 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
     @staticmethod
     def write_temp_run_config_file():
 
-        sys_config_path = IntegrationTestUtils.DEFAULT_SYS_CONFIG_FILE_PATH
-        sys_config_dict: dict = IntegrationTestUtils.load_json_file(sys_config_path)
+        sys_config = IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH
+        sys_config_dict: dict = parse_json_file_with_tokens(sys_config)
         sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
         sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
-        IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
-        IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH)
 
-        default_config = IntegrationTestUtils.DEFAULT_ENV_SUIT_CONFIG_FILE_PATH
-        default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
-        run_dict: dict = copy.deepcopy(default_config_dict)
+        os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
+        job_sys_config_file = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+        IntegrationTestUtils.write_json_file(sys_config_dict, job_sys_config_file)
+
+        # now set the sys config file path to the new job-specific file
+        IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH = job_sys_config_file
+
+        run_config = IntegrationTestUtils.RUN_CONFIG_FILE_PATH
+        run_config_temp_dict: dict = IntegrationTestUtils.load_json_file(run_config)
+        run_dict: dict = copy.deepcopy(run_config_temp_dict)
         run_dict['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
         run_dict['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
         run_dict['OUTPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
         run_dict['EWS-Plotting']['Ethiopia']['FilterForCountry'] = "True"
         run_dict['EWS-Plotting']['Kenya']['FilterForCountry'] = "True"
 
-        os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
-        IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
         IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)