From 6e358f62e27207e383a56121afa4e82beb46a8ad Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Fri, 10 May 2024 13:04:31 +0100
Subject: [PATCH] making tests work
---
tests/integration/full/full_test_advisory.py | 14 ++----
.../integration/full/full_test_deposition.py | 4 +-
tests/integration/full/full_test_env_suit.py | 4 +-
tests/integration/full/full_test_epi.py | 12 ++---
tests/integration/full/full_test_survey.py | 5 +--
.../partial/integration_test_utils.py | 7 ++-
.../configs/config_EastAfrica_fc_live.json | 44 +++++++++++++++++--
7 files changed, 55 insertions(+), 35 deletions(-)
diff --git a/tests/integration/full/full_test_advisory.py b/tests/integration/full/full_test_advisory.py
index 16c9335..c7809d8 100644
--- a/tests/integration/full/full_test_advisory.py
+++ b/tests/integration/full/full_test_advisory.py
@@ -2,10 +2,10 @@ import copy
import os
import sys
-from ProcessorAdvisory import ProcessorAdvisory
-from ProcessorDeposition import ProcessorDeposition
-from ProcessorEnvironment import ProcessorEnvironment
-from ProcessorSurveys import ProcessorSurveys
+from coordinator.ProcessorAdvisory import ProcessorAdvisory
+from coordinator.ProcessorDeposition import ProcessorDeposition
+from coordinator.ProcessorEnvironment import ProcessorEnvironment
+from coordinator.ProcessorSurveys import ProcessorSurveys
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.advisory_test_suite import BaseAdvisoryTestSuite
@@ -61,9 +61,6 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
@staticmethod
def run_dependent_pipelines():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
depo_processor = ProcessorDeposition()
IntegrationTestUtils.run_external_pipeline("Deposition",
IntegrationTestUtils.TEST_START_DATE,
@@ -83,9 +80,6 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
@staticmethod
def run_advisory_pipeline():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
advisory_processor = ProcessorAdvisory()
IntegrationTestUtils.run_external_pipeline(BaseAdvisoryTestSuite.AdvisoryTestSuite.ADVISORY_COMPONENT_NAME,
IntegrationTestUtils.TEST_START_DATE,
diff --git a/tests/integration/full/full_test_deposition.py b/tests/integration/full/full_test_deposition.py
index 14d1d11..0bb5a31 100644
--- a/tests/integration/full/full_test_deposition.py
+++ b/tests/integration/full/full_test_deposition.py
@@ -2,7 +2,7 @@ import copy
import os
import sys
-from ProcessorDeposition import ProcessorDeposition
+from coordinator.ProcessorDeposition import ProcessorDeposition
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.depo_test_suite import BaseDepoTestSuite
@@ -46,8 +46,6 @@ class FullTestDeposition(BaseDepoTestSuite.DepoTestSuite):
@staticmethod
def run_depo_pipeline():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
depo_processor = ProcessorDeposition()
IntegrationTestUtils.run_external_pipeline(BaseDepoTestSuite.DepoTestSuite.DEPO_COMPONENT_NAME,
IntegrationTestUtils.TEST_START_DATE,
diff --git a/tests/integration/full/full_test_env_suit.py b/tests/integration/full/full_test_env_suit.py
index 6196364..229168f 100644
--- a/tests/integration/full/full_test_env_suit.py
+++ b/tests/integration/full/full_test_env_suit.py
@@ -2,7 +2,7 @@ import copy
import os
import sys
-from ProcessorEnvironment import ProcessorEnvironment
+from coordinator.ProcessorEnvironment import ProcessorEnvironment
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.env_suit_test_suite import BaseEnvSuitTestSuite
@@ -47,8 +47,6 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
@staticmethod
def run_env_pipeline():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
env_processor = ProcessorEnvironment()
IntegrationTestUtils.run_external_pipeline(BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_COMPONENT_NAME,
IntegrationTestUtils.TEST_START_DATE,
diff --git a/tests/integration/full/full_test_epi.py b/tests/integration/full/full_test_epi.py
index 9389f26..17569d1 100644
--- a/tests/integration/full/full_test_epi.py
+++ b/tests/integration/full/full_test_epi.py
@@ -2,9 +2,9 @@ import copy
import os
import sys
-from ProcessorDeposition import ProcessorDeposition
-from ProcessorEnvironment import ProcessorEnvironment
-from ProcessorEpidemiology import ProcessorEpidemiology
+from coordinator.ProcessorDeposition import ProcessorDeposition
+from coordinator.ProcessorEnvironment import ProcessorEnvironment
+from coordinator.ProcessorEpidemiology import ProcessorEpidemiology
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.epi_test_suite import BaseEpiTestSuite
@@ -68,9 +68,6 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
@staticmethod
def run_dependent_pipelines():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
depo_processor = ProcessorDeposition()
IntegrationTestUtils.run_external_pipeline("Deposition",
IntegrationTestUtils.TEST_START_DATE,
@@ -84,9 +81,6 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
@staticmethod
def run_epi_pipeline():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
epi_processor = ProcessorEpidemiology()
IntegrationTestUtils.run_external_pipeline(BaseEpiTestSuite.EpiTestSuite.EPI_COMPONENT_NAME,
IntegrationTestUtils.TEST_START_DATE,
diff --git a/tests/integration/full/full_test_survey.py b/tests/integration/full/full_test_survey.py
index 157d6cc..2c1b3cc 100644
--- a/tests/integration/full/full_test_survey.py
+++ b/tests/integration/full/full_test_survey.py
@@ -2,7 +2,7 @@ import copy
import os
import sys
-from ProcessorSurveys import ProcessorSurveys
+from coordinator.ProcessorSurveys import ProcessorSurveys
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.survey_test_suite import BaseSurveyTestSuite
@@ -41,9 +41,6 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
@staticmethod
def run_survey_pipeline():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
survey_processor = ProcessorSurveys()
IntegrationTestUtils.run_external_pipeline(BaseSurveyTestSuite.SurveyTestSuite.SURVEY_COMPONENT_NAME,
IntegrationTestUtils.TEST_START_DATE,
diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py
index d9fb9f7..c32955f 100644
--- a/tests/integration/partial/integration_test_utils.py
+++ b/tests/integration/partial/integration_test_utils.py
@@ -95,8 +95,6 @@ class IntegrationTestUtils:
f"{processor_dir}_" +
IntegrationTestUtils.TEST_START_DATE)
- # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
- # sys.argv[1:] = _args.unittest_args
tests: TestSuite = TestLoader().loadTestsFromTestCase(test_case)
if _test_report_dir is None:
@@ -235,6 +233,11 @@ class IntegrationTestUtils:
for key, value in kwargs.items():
args_dict[key] = value
+ # need EMAIL_CRED in the environment before we run a Processor
+ os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
+ # need LOGGING_CONFIG_PATH in the environment before we run a Processor
+ os.environ["LOGGING_CONFIG"] = IntegrationTestUtils.LOGGING_CONFIG_PATH
+
try:
processor.run_process(args_dict)
except SystemExit:
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
index 86a99e4..abfbbf8 100644
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
@@ -35,15 +35,35 @@
"surveyor_infromation-country" : ["Kenya", "Ethiopia"]
}
},
- "WRSIS" : {}
+ "a5jrp2wkzpZ6am3aF5xGSo" : {
+ "add" : {
+ "Origin" : "newODK",
+ "PublishedLevel" : "Raw"
+ }
+ },
+ "aE6GkrJ9gVCV7kzE7gSeY2" : {
+ "add" : {
+ "Origin" : "newODK2",
+ "PublishedLevel" : "Raw"
+ }
+ }
+
},
"Groups" : {
"PROD" : {
- "Origin" : ["ODK-server", "kobo-server"]
+ "Origin" : ["CSV_01", "CSV_02", "ODK_01", "ODK_02", "CSV-CAM"]
+ },
+ "WRT-Publ" : {
+ "Origin" : ["CSV_01", "CSV_02", "ODK_01", "ODK_02"],
+ "PublishedLevel" : ["Publ"]
+ },
+ "ODK" : {
+ "Origin" : ["ODK-server", "kobo-server", "newODK", "newODK2", "CSV-CAM"]
}
},
- "GroupBy" : ["Origin"],
- "GroupsToIgnore" : ["ODK-server", "kobo-server", "newODK", "newODK2", "CSV-CAM"],
+ "GroupBy" : ["Origin", "PublishedLevel"],
+ "GroupsToIgnore" : ["ODK-server", "kobo-server", "newODK", "newODK2", "CSV-CAM", "CSV_01", "CSV_02", "ODK_01", "ODK_02", "Unpubl", "Publ", "Raw"],
+ "ODKDatabasePathTemplate" : "${WorkspacePathout}/ODK_DB/",
"SurveyFormat" : "ODK",
"SurveyorNameCol" : "surveyor_infromation-surveyor_name",
"SourcesRegionName" : "EastAfrica",
@@ -193,6 +213,22 @@
"FileNameTemplate" : "RIE_value.nc",
"FileNamePrepared" : "?"
},
+ "Env" : {
+ "SuccessFileTemplate" : "${WorkspacePath}EPI_${YesterdayString}/STATUS_SUCCESS",
+ "PathTemplate" : "${WorkspacePath}EPI_${YesterdayString}/${RegionName}/${DiseaseName}/",
+ "FileNameTemplate" : "infections_config_EastAfrica_fc_live_Epidemiology_*_env_progression.csv",
+ "FileListerFunction" : "list_onefile_historical",
+ "FileLoaderFunction" : "load_and_restructure_epi_file",
+ "FileNamePrepared" : "?"
+ },
+ "ps" : {
+ "SuccessFileTemplate" : "${WorkspacePath}EPI_${YesterdayString}/STATUS_SUCCESS",
+ "PathTemplate" : "${WorkspacePath}EPI_${YesterdayString}/${RegionName}/${DiseaseName}/",
+ "FileNameTemplate" : "infections_config_EastAfrica_fc_live_Epidemiology_*_psbeta0.004gamma0.00025alpha1.0_progression.csv",
+ "FileListerFunction" : "list_onefile_historical",
+ "FileLoaderFunction" : "load_and_restructure_epi_file",
+ "FileNamePrepared" : "?"
+ },
"Epi" : [
{
"model" : "Env",
--
GitLab