fixing tests

.gitlab-ci.yml

@@ -244,9 +244,8 @@ full_fat_epi:
       --outdir ${TEST_WORKSPACE_DIR}/integration/full/
       --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
       --test_report_dir $CI_PROJECT_DIR/test_reports
-      --run_date_type $FULL_FAT_RUN_DATE_TYPE
       --custom_run_date $CUSTOM_RUN_DATE
-      --custom_dir_prefix temp_advisory_$RUN_TIME
+      --custom_dir_prefix temp_epi_$RUN_TIME
 
       python3 full_test_env_suit.py 
       --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
@@ -254,9 +253,8 @@ full_fat_epi:
       --outdir ${TEST_WORKSPACE_DIR}/integration/full/
       --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
       --test_report_dir $CI_PROJECT_DIR/test_reports
-      --run_date_type $FULL_FAT_RUN_DATE_TYPE
       --custom_run_date $CUSTOM_RUN_DATE
-      --custom_dir_prefix temp_advisory_$RUN_TIME
+      --custom_dir_prefix temp_epi_$RUN_TIME
 
       python3 full_test_epi.py 
       --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
@@ -264,9 +262,8 @@ full_fat_epi:
       --outdir ${TEST_WORKSPACE_DIR}/integration/full/
       --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
       --test_report_dir $CI_PROJECT_DIR/test_reports
-      --run_date_type $FULL_FAT_RUN_DATE_TYPE
       --custom_run_date $CUSTOM_RUN_DATE
-      --custom_dir_prefix temp_advisory_$RUN_TIME
+      --custom_dir_prefix temp_epi_$RUN_TIME
   artifacts:
     paths:
       - $CI_PROJECT_DIR/test_reports

tests/integration/full/full_test_advisory.py

@@ -3,15 +3,9 @@ import os
 import sys
 
 from ews.coordinator.processor_advisory import ProcessorAdvisory
-from ews.coordinator.processor_deposition import ProcessorDeposition
-from ews.coordinator.processor_environment import ProcessorEnvironment
-from ews.coordinator.processor_surveys import ProcessorSurveys
 from ews.coordinator.utils.processor_utils import parse_json_file_with_tokens
 from integration.partial.integration_test_utils import IntegrationTestUtils
 from integration.test_suites.advisory_test_suite import BaseAdvisoryTestSuite
-from integration.test_suites.depo_test_suite import BaseDepoTestSuite
-from integration.test_suites.env_suit_test_suite import BaseEnvSuitTestSuite
-from integration.test_suites.survey_test_suite import BaseSurveyTestSuite
 
 
 class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
@@ -30,7 +24,6 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
                 or not os.path.exists(IntegrationTestUtils.TEST_JOB_DIR):
             # if True:
             FullTestAdvisory.write_temp_run_config_files()
-            # FullTestAdvisory.run_dependent_pipelines()
             FullTestAdvisory.run_advisory_pipeline()
         else:
             string = f"output in {IntegrationTestUtils.TEST_JOB_DIR} already written, skipping rerun"
@@ -41,17 +34,6 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
     @staticmethod
     def write_temp_run_config_files():
 
-        # default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
-        # default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
-        # run_dict: dict = copy.deepcopy(default_config_dict)
-        # run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
-        # run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
-        #
-        # # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
-        # os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
-        # IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
-        # IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)
-
         """
         IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH will have been set from the command line args, and point to the
         production sys config file. We need to modify this file to point to the test output directory.
@@ -80,28 +62,6 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
         # set the run config file path to the new job-specific file
         IntegrationTestUtils.RUN_CONFIG_FILE_PATH = job_run_config_file
 
-    @staticmethod
-    def run_dependent_pipelines():
-        depo_processor = ProcessorDeposition()
-        IntegrationTestUtils.run_external_pipeline(BaseDepoTestSuite.DepoTestSuite.DEPO_COMPONENT_NAME,
-                                                   BaseDepoTestSuite.DepoTestSuite.DEPO_PROCESSOR_DIR,
-                                                   IntegrationTestUtils.TEST_START_DATE,
-                                                   depo_processor)
-
-        env_processor = ProcessorEnvironment()
-        IntegrationTestUtils.run_external_pipeline(BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_COMPONENT_NAME,
-                                                   BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_PROCESSOR_DIR,
-                                                   IntegrationTestUtils.TEST_START_DATE,
-                                                   env_processor)
-
-        survey_processor = ProcessorSurveys()
-        previous_day_string: str = IntegrationTestUtils.get_day_before_as_string(IntegrationTestUtils.TEST_START_DATE)
-        IntegrationTestUtils.run_external_pipeline(BaseSurveyTestSuite.SurveyTestSuite.SURVEY_COMPONENT_NAME,
-                                                   BaseSurveyTestSuite.SurveyTestSuite.SURVEY_PROCESSOR_DIR,
-                                                   previous_day_string,
-                                                   survey_processor)
-        pass
-
     @staticmethod
     def run_advisory_pipeline():
         advisory_processor = ProcessorAdvisory()

tests/integration/partial/integration_test_utils.py

@@ -123,12 +123,14 @@ class IntegrationTestUtils:
         tests: TestSuite = TestLoader().loadTestsFromTestCase(test_case)
 
         if _test_report_dir is None:
-            _test_report_dir = IntegrationTestUtils.TEST_JOB_DIR
+            _test_report_dir = IntegrationTestUtils.TEST_OUT_PATH
 
         """
         the HTMLTestRunner will create the directory if it does not exist, we are putting the output into the 
-        TEST_OUT_PATH, which is the top-level test directory for this run, not the TEST_JOB_DIR, which is the dir
-        for the pipeline being tested
+        _test_report_dir, which defaults to TEST_OUT_PATH, which is the top-level test directory for this run, not the 
+        TEST_JOB_DIR, which is the dir for the pipeline being tested. We need to make sure the TEST_JOB_DIR does not
+        exist before the test is instantiated, to ensure the initial pipeline run takes place. Setting the test report
+        to go into the TEST_JOB_DIR will create the parent dir and stop the pipeline from running
         """
         runner = HTMLTestRunner(output = _test_report_dir, log = True, report_name = f"{test_prefix}_results")
         result: TestResult = runner.run(tests)

tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/survey_config_EastAfrica_fc_live.json

@@ -69,7 +69,7 @@
     ],
     "ODKDatabasePathTemplate": "${WorkspacePathout}/ODK_DB/",
     "SurveyFormat": "WRT",
-    "SurveyorNameCol" : "surveyor_infromation-surveyor_name",
+    "SurveyorNameCol" : "NOT_USED?",
     "SourcesRegionName" : "EastAfrica",
     "SourcesConfigs": {
         "WRT-Publ": "${ConfigsPath}/source_gen/config_EastAfrica_mapspam2017.json"