making tests work

tests/integration/full/full_test_advisory.py

@@ -2,10 +2,10 @@ import copy
 import os
 import sys
 
-from ProcessorAdvisory import ProcessorAdvisory
-from ProcessorDeposition import ProcessorDeposition
-from ProcessorEnvironment import ProcessorEnvironment
-from ProcessorSurveys import ProcessorSurveys
+from coordinator.ProcessorAdvisory import ProcessorAdvisory
+from coordinator.ProcessorDeposition import ProcessorDeposition
+from coordinator.ProcessorEnvironment import ProcessorEnvironment
+from coordinator.ProcessorSurveys import ProcessorSurveys
 from integration.partial.integration_test_utils import IntegrationTestUtils
 from integration.test_suites.advisory_test_suite import BaseAdvisoryTestSuite
 
@@ -61,9 +61,6 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
 
     @staticmethod
     def run_dependent_pipelines():
-        #  need EMAIL_CRED in the environment before we import Processor
-        os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
         depo_processor = ProcessorDeposition()
         IntegrationTestUtils.run_external_pipeline("Deposition",
                                                    IntegrationTestUtils.TEST_START_DATE,
@@ -83,9 +80,6 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
 
     @staticmethod
     def run_advisory_pipeline():
-        #  need EMAIL_CRED in the environment before we import Processor
-        os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
         advisory_processor = ProcessorAdvisory()
         IntegrationTestUtils.run_external_pipeline(BaseAdvisoryTestSuite.AdvisoryTestSuite.ADVISORY_COMPONENT_NAME,
                                                    IntegrationTestUtils.TEST_START_DATE,

tests/integration/full/full_test_deposition.py

@@ -2,7 +2,7 @@ import copy
 import os
 import sys
 
-from ProcessorDeposition import ProcessorDeposition
+from coordinator.ProcessorDeposition import ProcessorDeposition
 from integration.partial.integration_test_utils import IntegrationTestUtils
 from integration.test_suites.depo_test_suite import BaseDepoTestSuite
 
@@ -46,8 +46,6 @@ class FullTestDeposition(BaseDepoTestSuite.DepoTestSuite):
 
     @staticmethod
     def run_depo_pipeline():
-        #  need EMAIL_CRED in the environment before we import Processor
-        os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
         depo_processor = ProcessorDeposition()
         IntegrationTestUtils.run_external_pipeline(BaseDepoTestSuite.DepoTestSuite.DEPO_COMPONENT_NAME,
                                                    IntegrationTestUtils.TEST_START_DATE,

tests/integration/full/full_test_env_suit.py

@@ -2,7 +2,7 @@ import copy
 import os
 import sys
 
-from ProcessorEnvironment import ProcessorEnvironment
+from coordinator.ProcessorEnvironment import ProcessorEnvironment
 from integration.partial.integration_test_utils import IntegrationTestUtils
 from integration.test_suites.env_suit_test_suite import BaseEnvSuitTestSuite
 
@@ -47,8 +47,6 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
 
     @staticmethod
     def run_env_pipeline():
-        #  need EMAIL_CRED in the environment before we import Processor
-        os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
         env_processor = ProcessorEnvironment()
         IntegrationTestUtils.run_external_pipeline(BaseEnvSuitTestSuite.EnvSuitTestSuite.ENV_COMPONENT_NAME,
                                                    IntegrationTestUtils.TEST_START_DATE,

tests/integration/full/full_test_epi.py

@@ -2,9 +2,9 @@ import copy
 import os
 import sys
 
-from ProcessorDeposition import ProcessorDeposition
-from ProcessorEnvironment import ProcessorEnvironment
-from ProcessorEpidemiology import ProcessorEpidemiology
+from coordinator.ProcessorDeposition import ProcessorDeposition
+from coordinator.ProcessorEnvironment import ProcessorEnvironment
+from coordinator.ProcessorEpidemiology import ProcessorEpidemiology
 from integration.partial.integration_test_utils import IntegrationTestUtils
 from integration.test_suites.epi_test_suite import BaseEpiTestSuite
 
@@ -68,9 +68,6 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
 
     @staticmethod
     def run_dependent_pipelines():
-        #  need EMAIL_CRED in the environment before we import Processor
-        os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
         depo_processor = ProcessorDeposition()
         IntegrationTestUtils.run_external_pipeline("Deposition",
                                                    IntegrationTestUtils.TEST_START_DATE,
@@ -84,9 +81,6 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
 
     @staticmethod
     def run_epi_pipeline():
-        #  need EMAIL_CRED in the environment before we import Processor
-        os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
         epi_processor = ProcessorEpidemiology()
         IntegrationTestUtils.run_external_pipeline(BaseEpiTestSuite.EpiTestSuite.EPI_COMPONENT_NAME,
                                                    IntegrationTestUtils.TEST_START_DATE,

tests/integration/full/full_test_survey.py

@@ -2,7 +2,7 @@ import copy
 import os
 import sys
 
-from ProcessorSurveys import ProcessorSurveys
+from coordinator.ProcessorSurveys import ProcessorSurveys
 from integration.partial.integration_test_utils import IntegrationTestUtils
 from integration.test_suites.survey_test_suite import BaseSurveyTestSuite
 
@@ -41,9 +41,6 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
 
     @staticmethod
     def run_survey_pipeline():
-        #  need EMAIL_CRED in the environment before we import Processor
-        os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
-
         survey_processor = ProcessorSurveys()
         IntegrationTestUtils.run_external_pipeline(BaseSurveyTestSuite.SurveyTestSuite.SURVEY_COMPONENT_NAME,
                                                    IntegrationTestUtils.TEST_START_DATE,

tests/integration/partial/integration_test_utils.py

@@ -95,8 +95,6 @@ class IntegrationTestUtils:
                                                          f"{processor_dir}_" +
                                                          IntegrationTestUtils.TEST_START_DATE)
 
-        #  Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
-        # sys.argv[1:] = _args.unittest_args
         tests: TestSuite = TestLoader().loadTestsFromTestCase(test_case)
 
         if _test_report_dir is None:
@@ -235,6 +233,11 @@ class IntegrationTestUtils:
         for key, value in kwargs.items():
             args_dict[key] = value
 
+        #  need EMAIL_CRED in the environment before we run a Processor
+        os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
+        #  need LOGGING_CONFIG_PATH in the environment before we run a Processor
+        os.environ["LOGGING_CONFIG"] = IntegrationTestUtils.LOGGING_CONFIG_PATH
+
         try:
             processor.run_process(args_dict)
         except SystemExit:

tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json

@@ -35,15 +35,35 @@
                     "surveyor_infromation-country" : ["Kenya", "Ethiopia"]
                 }
             },
-            "WRSIS" : {}
+            "a5jrp2wkzpZ6am3aF5xGSo" : {
+                "add" : {
+                    "Origin" : "newODK",
+                    "PublishedLevel" : "Raw"
+                }
+            },
+            "aE6GkrJ9gVCV7kzE7gSeY2" : {
+                "add" : {
+                    "Origin" : "newODK2",
+                    "PublishedLevel" : "Raw"
+                }
+            }
+
         },
         "Groups" : {
             "PROD" : {
-                "Origin" : ["ODK-server", "kobo-server"]
+                "Origin" : ["CSV_01", "CSV_02", "ODK_01", "ODK_02", "CSV-CAM"]
+            },
+            "WRT-Publ" : {
+                "Origin" : ["CSV_01", "CSV_02", "ODK_01", "ODK_02"],
+                "PublishedLevel" : ["Publ"]
+        },
+                        "ODK" : {
+                "Origin" : ["ODK-server", "kobo-server", "newODK", "newODK2", "CSV-CAM"]
             }
         },
-        "GroupBy" : ["Origin"],
-        "GroupsToIgnore" : ["ODK-server", "kobo-server", "newODK", "newODK2", "CSV-CAM"],
+        "GroupBy" : ["Origin", "PublishedLevel"],
+        "GroupsToIgnore" : ["ODK-server", "kobo-server", "newODK", "newODK2", "CSV-CAM", "CSV_01", "CSV_02", "ODK_01", "ODK_02", "Unpubl", "Publ", "Raw"],
+        "ODKDatabasePathTemplate" : "${WorkspacePathout}/ODK_DB/",
         "SurveyFormat" : "ODK",
         "SurveyorNameCol" : "surveyor_infromation-surveyor_name",
         "SourcesRegionName" : "EastAfrica",
@@ -193,6 +213,22 @@
             "FileNameTemplate" : "RIE_value.nc",
             "FileNamePrepared" : "?"
         },
+	"Env" : {
+            "SuccessFileTemplate" : "${WorkspacePath}EPI_${YesterdayString}/STATUS_SUCCESS",
+            "PathTemplate" : "${WorkspacePath}EPI_${YesterdayString}/${RegionName}/${DiseaseName}/",
+            "FileNameTemplate" : "infections_config_EastAfrica_fc_live_Epidemiology_*_env_progression.csv",
+            "FileListerFunction" : "list_onefile_historical",
+            "FileLoaderFunction" : "load_and_restructure_epi_file",
+            "FileNamePrepared" : "?"
+        },
+        "ps" : {
+            "SuccessFileTemplate" : "${WorkspacePath}EPI_${YesterdayString}/STATUS_SUCCESS",
+            "PathTemplate" : "${WorkspacePath}EPI_${YesterdayString}/${RegionName}/${DiseaseName}/",
+            "FileNameTemplate" : "infections_config_EastAfrica_fc_live_Epidemiology_*_psbeta0.004gamma0.00025alpha1.0_progression.csv",
+            "FileListerFunction" : "list_onefile_historical",
+            "FileLoaderFunction" : "load_and_restructure_epi_file",
+            "FileNamePrepared" : "?"
+        },
         "Epi" : [
             {
                 "model" : "Env",