From 4b47d99f809a16d7415ea33336472a2404b4abb6 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Fri, 12 Jul 2024 15:50:07 +0100
Subject: [PATCH] fixing tests
---
.gitlab-ci.yml | 9 ++---
tests/integration/full/full_test_advisory.py | 40 -------------------
.../partial/integration_test_utils.py | 8 ++--
.../survey_config_EastAfrica_fc_live.json | 2 +-
4 files changed, 9 insertions(+), 50 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index be953c9..c54011e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -244,9 +244,8 @@ full_fat_epi:
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--test_report_dir $CI_PROJECT_DIR/test_reports
- --run_date_type $FULL_FAT_RUN_DATE_TYPE
--custom_run_date $CUSTOM_RUN_DATE
- --custom_dir_prefix temp_advisory_$RUN_TIME
+ --custom_dir_prefix temp_epi_$RUN_TIME
python3 full_test_env_suit.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
@@ -254,9 +253,8 @@ full_fat_epi:
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--test_report_dir $CI_PROJECT_DIR/test_reports
- --run_date_type $FULL_FAT_RUN_DATE_TYPE
--custom_run_date $CUSTOM_RUN_DATE
- --custom_dir_prefix temp_advisory_$RUN_TIME
+ --custom_dir_prefix temp_epi_$RUN_TIME
python3 full_test_epi.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
@@ -264,9 +262,8 @@ full_fat_epi:
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--test_report_dir $CI_PROJECT_DIR/test_reports
- --run_date_type $FULL_FAT_RUN_DATE_TYPE
--custom_run_date $CUSTOM_RUN_DATE
- --custom_dir_prefix temp_advisory_$RUN_TIME
+ --custom_dir_prefix temp_epi_$RUN_TIME
artifacts:
paths:
- $CI_PROJECT_DIR/test_reports
diff --git a/tests/integration/full/full_test_advisory.py b/tests/integration/full/full_test_advisory.py
index 1e9e235..2fe4a1b 100644
--- a/tests/integration/full/full_test_advisory.py
+++ b/tests/integration/full/full_test_advisory.py
@@ -3,15 +3,9 @@ import os
import sys
from ews.coordinator.processor_advisory import ProcessorAdvisory
-from ews.coordinator.processor_deposition import ProcessorDeposition
-from ews.coordinator.processor_environment import ProcessorEnvironment
-from ews.coordinator.processor_surveys import ProcessorSurveys
from ews.coordinator.utils.processor_utils import parse_json_file_with_tokens
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.advisory_test_suite import BaseAdvisoryTestSuite
-from integration.test_suites.depo_test_suite import BaseDepoTestSuite
-from integration.test_suites.env_suit_test_suite import BaseEnvSuitTestSuite
-from integration.test_suites.survey_test_suite import BaseSurveyTestSuite
class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
@@ -30,7 +24,6 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
or not os.path.exists(IntegrationTestUtils.TEST_JOB_DIR):
# if True:
FullTestAdvisory.write_temp_run_config_files()
- # FullTestAdvisory.run_dependent_pipelines()
FullTestAdvisory.run_advisory_pipeline()
else:
string = f"output in {IntegrationTestUtils.TEST_JOB_DIR} already written, skipping rerun"
@@ -41,17 +34,6 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
@staticmethod
def write_temp_run_config_files():
- # default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
- # default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
- # run_dict: dict = copy.deepcopy(default_config_dict)
- # run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
- # run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
- #
- # # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
- # os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- # IntegrationTestUtils.RUN_CONFIG_FILE_PATH = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- # IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.RUN_CONFIG_FILE_PATH)
-
"""
IntegrationTestUtils.RUN_SYS_CONFIG_FILE_PATH will have been set from the command line args, and point to the
production sys config file. We need to modify this file to point to the test output directory.
@@ -80,28 +62,6 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
# set the run config file path to the new job-specific file
IntegrationTestUtils.RUN_CONFIG_FILE_PATH = job_run_config_file
- @staticmethod
- def run_dependent_pipelines():
- depo_processor = ProcessorDeposition()
- IntegrationTestUtils.run_external_pipeline(BaseDepoTestSuite.DepoTestSuite.DEPO_COMPONENT_NAME,
- BaseDepoTestSuite.DepoTestSuite.DEPO_PROCESSOR_DIR,
- IntegrationTestUtils.TEST_START_DATE,
- depo_processor)
-
- env_processor = ProcessorEnvironment()
- IntegrationTestUtils.run_external_pipeline(BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_COMPONENT_NAME,
- BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_PROCESSOR_DIR,
- IntegrationTestUtils.TEST_START_DATE,
- env_processor)
-
- survey_processor = ProcessorSurveys()
- previous_day_string: str = IntegrationTestUtils.get_day_before_as_string(IntegrationTestUtils.TEST_START_DATE)
- IntegrationTestUtils.run_external_pipeline(BaseSurveyTestSuite.SurveyTestSuite.SURVEY_COMPONENT_NAME,
- BaseSurveyTestSuite.SurveyTestSuite.SURVEY_PROCESSOR_DIR,
- previous_day_string,
- survey_processor)
- pass
-
@staticmethod
def run_advisory_pipeline():
advisory_processor = ProcessorAdvisory()
diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py
index 73b4f6c..54e48dd 100644
--- a/tests/integration/partial/integration_test_utils.py
+++ b/tests/integration/partial/integration_test_utils.py
@@ -123,12 +123,14 @@ class IntegrationTestUtils:
tests: TestSuite = TestLoader().loadTestsFromTestCase(test_case)
if _test_report_dir is None:
- _test_report_dir = IntegrationTestUtils.TEST_JOB_DIR
+ _test_report_dir = IntegrationTestUtils.TEST_OUT_PATH
"""
the HTMLTestRunner will create the directory if it does not exist, we are putting the output into the
- TEST_OUT_PATH, which is the top-level test directory for this run, not the TEST_JOB_DIR, which is the dir
- for the pipeline being tested
+ _test_report_dir, which defaults to TEST_OUT_PATH, which is the top-level test directory for this run, not the
+ TEST_JOB_DIR, which is the dir for the pipeline being tested. We need to make sure the TEST_JOB_DIR does not
+ exist before the test is instantiated, to ensure the initial pipeline run takes place. Setting the test report
+ to go into the TEST_JOB_DIR will create the parent dir and stop the pipeline from running
"""
runner = HTMLTestRunner(output = _test_report_dir, log = True, report_name = f"{test_prefix}_results")
result: TestResult = runner.run(tests)
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/survey_config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/survey_config_EastAfrica_fc_live.json
index 5e50015..5133d04 100644
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/survey_config_EastAfrica_fc_live.json
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/survey_config_EastAfrica_fc_live.json
@@ -69,7 +69,7 @@
],
"ODKDatabasePathTemplate": "${WorkspacePathout}/ODK_DB/",
"SurveyFormat": "WRT",
- "SurveyorNameCol" : "surveyor_infromation-surveyor_name",
+ "SurveyorNameCol" : "NOT_USED?",
"SourcesRegionName" : "EastAfrica",
"SourcesConfigs": {
"WRT-Publ": "${ConfigsPath}/source_gen/config_EastAfrica_mapspam2017.json"
--
GitLab