diff --git a/tests/integration/full/full_test_advisory.py b/tests/integration/full/full_test_advisory.py
index 16c9335d3b41bf914fbf93fe9d58dcbf1ba71f59..c7809d884592d659642cc53b473c1797d09dfa1b 100644
--- a/tests/integration/full/full_test_advisory.py
+++ b/tests/integration/full/full_test_advisory.py
@@ -2,10 +2,10 @@ import copy
import os
import sys
-from ProcessorAdvisory import ProcessorAdvisory
-from ProcessorDeposition import ProcessorDeposition
-from ProcessorEnvironment import ProcessorEnvironment
-from ProcessorSurveys import ProcessorSurveys
+from coordinator.ProcessorAdvisory import ProcessorAdvisory
+from coordinator.ProcessorDeposition import ProcessorDeposition
+from coordinator.ProcessorEnvironment import ProcessorEnvironment
+from coordinator.ProcessorSurveys import ProcessorSurveys
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.advisory_test_suite import BaseAdvisoryTestSuite
@@ -61,9 +61,6 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
@staticmethod
def run_dependent_pipelines():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
depo_processor = ProcessorDeposition()
IntegrationTestUtils.run_external_pipeline("Deposition",
IntegrationTestUtils.TEST_START_DATE,
@@ -83,9 +80,6 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
@staticmethod
def run_advisory_pipeline():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
advisory_processor = ProcessorAdvisory()
IntegrationTestUtils.run_external_pipeline(BaseAdvisoryTestSuite.AdvisoryTestSuite.ADVISORY_COMPONENT_NAME,
IntegrationTestUtils.TEST_START_DATE,
diff --git a/tests/integration/full/full_test_deposition.py b/tests/integration/full/full_test_deposition.py
index 14d1d11d5532e321270897084766af0dbf9f78d7..0bb5a315d4240c188cecca020e83545a7d5eafca 100644
--- a/tests/integration/full/full_test_deposition.py
+++ b/tests/integration/full/full_test_deposition.py
@@ -2,7 +2,7 @@ import copy
import os
import sys
-from ProcessorDeposition import ProcessorDeposition
+from coordinator.ProcessorDeposition import ProcessorDeposition
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.depo_test_suite import BaseDepoTestSuite
@@ -46,8 +46,6 @@ class FullTestDeposition(BaseDepoTestSuite.DepoTestSuite):
@staticmethod
def run_depo_pipeline():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
depo_processor = ProcessorDeposition()
IntegrationTestUtils.run_external_pipeline(BaseDepoTestSuite.DepoTestSuite.DEPO_COMPONENT_NAME,
IntegrationTestUtils.TEST_START_DATE,
diff --git a/tests/integration/full/full_test_env_suit.py b/tests/integration/full/full_test_env_suit.py
index 619636474bd268c362db0c1dcf31b13403d252b4..229168f3bacdd20c0743e1e385d8a1ec174c9ab4 100644
--- a/tests/integration/full/full_test_env_suit.py
+++ b/tests/integration/full/full_test_env_suit.py
@@ -2,7 +2,7 @@ import copy
import os
import sys
-from ProcessorEnvironment import ProcessorEnvironment
+from coordinator.ProcessorEnvironment import ProcessorEnvironment
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.env_suit_test_suite import BaseEnvSuitTestSuite
@@ -47,8 +47,6 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
@staticmethod
def run_env_pipeline():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
env_processor = ProcessorEnvironment()
IntegrationTestUtils.run_external_pipeline(BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_COMPONENT_NAME,
IntegrationTestUtils.TEST_START_DATE,
diff --git a/tests/integration/full/full_test_epi.py b/tests/integration/full/full_test_epi.py
index 9389f26db1bb9e1afbc47f9036805e5e12f1a3d7..17569d142aacdf49c02bd0f1db74a2d0e2cc2608 100644
--- a/tests/integration/full/full_test_epi.py
+++ b/tests/integration/full/full_test_epi.py
@@ -2,9 +2,9 @@ import copy
import os
import sys
-from ProcessorDeposition import ProcessorDeposition
-from ProcessorEnvironment import ProcessorEnvironment
-from ProcessorEpidemiology import ProcessorEpidemiology
+from coordinator.ProcessorDeposition import ProcessorDeposition
+from coordinator.ProcessorEnvironment import ProcessorEnvironment
+from coordinator.ProcessorEpidemiology import ProcessorEpidemiology
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.epi_test_suite import BaseEpiTestSuite
@@ -68,9 +68,6 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
@staticmethod
def run_dependent_pipelines():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
depo_processor = ProcessorDeposition()
IntegrationTestUtils.run_external_pipeline("Deposition",
IntegrationTestUtils.TEST_START_DATE,
@@ -84,9 +81,6 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
@staticmethod
def run_epi_pipeline():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
epi_processor = ProcessorEpidemiology()
IntegrationTestUtils.run_external_pipeline(BaseEpiTestSuite.EpiTestSuite.EPI_COMPONENT_NAME,
IntegrationTestUtils.TEST_START_DATE,
diff --git a/tests/integration/full/full_test_survey.py b/tests/integration/full/full_test_survey.py
index 157d6cc607c84c18218efc4a479c62fb0531bbec..2c1b3ccc6baad5e6882a4fca9dad6a29fc51276f 100644
--- a/tests/integration/full/full_test_survey.py
+++ b/tests/integration/full/full_test_survey.py
@@ -2,7 +2,7 @@ import copy
import os
import sys
-from ProcessorSurveys import ProcessorSurveys
+from coordinator.ProcessorSurveys import ProcessorSurveys
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.survey_test_suite import BaseSurveyTestSuite
@@ -41,9 +41,6 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
@staticmethod
def run_survey_pipeline():
- # need EMAIL_CRED in the environment before we import Processor
- os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
survey_processor = ProcessorSurveys()
IntegrationTestUtils.run_external_pipeline(BaseSurveyTestSuite.SurveyTestSuite.SURVEY_COMPONENT_NAME,
IntegrationTestUtils.TEST_START_DATE,
diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py
index d9fb9f77e4ce7d95f5ca20282d44fb112ffa7882..c32955fc6b2a0a40690671d17014d7cfcab4c210 100644
--- a/tests/integration/partial/integration_test_utils.py
+++ b/tests/integration/partial/integration_test_utils.py
@@ -95,8 +95,6 @@ class IntegrationTestUtils:
f"{processor_dir}_" +
IntegrationTestUtils.TEST_START_DATE)
- # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
- # sys.argv[1:] = _args.unittest_args
tests: TestSuite = TestLoader().loadTestsFromTestCase(test_case)
if _test_report_dir is None:
@@ -235,6 +233,11 @@ class IntegrationTestUtils:
for key, value in kwargs.items():
args_dict[key] = value
+ # need EMAIL_CRED in the environment before we run a Processor
+ os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
+ # need LOGGING_CONFIG_PATH in the environment before we run a Processor
+ os.environ["LOGGING_CONFIG"] = IntegrationTestUtils.LOGGING_CONFIG_PATH
+
try:
processor.run_process(args_dict)
except SystemExit:
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
index 86a99e4134bbd1bc2a8a6de6836af8f7227b18a6..abfbbf826de9544f20612d29f0a98365ee335401 100644
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
@@ -35,15 +35,35 @@
"surveyor_infromation-country" : ["Kenya", "Ethiopia"]
}
},
- "WRSIS" : {}
+ "a5jrp2wkzpZ6am3aF5xGSo" : {
+ "add" : {
+ "Origin" : "newODK",
+ "PublishedLevel" : "Raw"
+ }
+ },
+ "aE6GkrJ9gVCV7kzE7gSeY2" : {
+ "add" : {
+ "Origin" : "newODK2",
+ "PublishedLevel" : "Raw"
+ }
+ }
+
},
"Groups" : {
"PROD" : {
- "Origin" : ["ODK-server", "kobo-server"]
+ "Origin" : ["CSV_01", "CSV_02", "ODK_01", "ODK_02", "CSV-CAM"]
+ },
+ "WRT-Publ" : {
+ "Origin" : ["CSV_01", "CSV_02", "ODK_01", "ODK_02"],
+ "PublishedLevel" : ["Publ"]
+ },
+ "ODK" : {
+ "Origin" : ["ODK-server", "kobo-server", "newODK", "newODK2", "CSV-CAM"]
}
},
- "GroupBy" : ["Origin"],
- "GroupsToIgnore" : ["ODK-server", "kobo-server", "newODK", "newODK2", "CSV-CAM"],
+ "GroupBy" : ["Origin", "PublishedLevel"],
+ "GroupsToIgnore" : ["ODK-server", "kobo-server", "newODK", "newODK2", "CSV-CAM", "CSV_01", "CSV_02", "ODK_01", "ODK_02", "Unpubl", "Publ", "Raw"],
+ "ODKDatabasePathTemplate" : "${WorkspacePathout}/ODK_DB/",
"SurveyFormat" : "ODK",
"SurveyorNameCol" : "surveyor_infromation-surveyor_name",
"SourcesRegionName" : "EastAfrica",
@@ -193,6 +213,22 @@
"FileNameTemplate" : "RIE_value.nc",
"FileNamePrepared" : "?"
},
+ "Env" : {
+ "SuccessFileTemplate" : "${WorkspacePath}EPI_${YesterdayString}/STATUS_SUCCESS",
+ "PathTemplate" : "${WorkspacePath}EPI_${YesterdayString}/${RegionName}/${DiseaseName}/",
+ "FileNameTemplate" : "infections_config_EastAfrica_fc_live_Epidemiology_*_env_progression.csv",
+ "FileListerFunction" : "list_onefile_historical",
+ "FileLoaderFunction" : "load_and_restructure_epi_file",
+ "FileNamePrepared" : "?"
+ },
+ "ps" : {
+ "SuccessFileTemplate" : "${WorkspacePath}EPI_${YesterdayString}/STATUS_SUCCESS",
+ "PathTemplate" : "${WorkspacePath}EPI_${YesterdayString}/${RegionName}/${DiseaseName}/",
+ "FileNameTemplate" : "infections_config_EastAfrica_fc_live_Epidemiology_*_psbeta0.004gamma0.00025alpha1.0_progression.csv",
+ "FileListerFunction" : "list_onefile_historical",
+ "FileLoaderFunction" : "load_and_restructure_epi_file",
+ "FileNamePrepared" : "?"
+ },
"Epi" : [
{
"model" : "Env",