repackaging integration tests

.gitlab-ci.yml

@@ -57,7 +57,7 @@ test:
     - python3 -m coverage run -m unittest *
     - python -m coverage report
     - python -m coverage html -d $CI_PROJECT_DIR/coverage
-    - cd $CI_PROJECT_DIR/tests/integration
+    - cd $CI_PROJECT_DIR/tests/integration/partial/
     - python3 -m coverage run -m unittest test_advisory.py
     - python3 -m coverage run -m unittest test_deposition.py
     - python3 -m coverage run -m unittest test_env_suit.py

configs/docker/build/build.sh

 cp ../../../conda-env-py3EWS-withbuilds.yml .;
-sudo docker build --target base_image -t ews_coordinator .;
+sudo docker build -t ews_coordinator .;
 rm -v conda-env-py3EWS-withbuilds.yml;

tests/integration/integration_test_utils.py → tests/integration/partial/integration_test_utils.py

@@ -8,12 +8,12 @@ from zipfile import ZipFile
 
 class IntegrationTestUtils:
 
-    EMAIL_CRED_PATH: str = "../test_data/test_deployment/envs/Cred_gmail.json"
-    DEFAULT_CONFIG_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json"
-    TEMP_CONFIG_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/workspace/temp_config.json"
-    EXAMPLE_SURVEY_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_survey_run.zip"
-    EXAMPLE_DEPO_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_depo_run.zip"
-    EXAMPLE_ENV_SUIT_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_env_suit_run.zip"
+    EMAIL_CRED_PATH: str = "../../test_data/test_deployment/envs/Cred_gmail.json"
+    DEFAULT_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json"
+    TEMP_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/workspace/temp_config.json"
+    EXAMPLE_SURVEY_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_survey_run.zip"
+    EXAMPLE_DEPO_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_depo_run.zip"
+    EXAMPLE_ENV_SUIT_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_env_suit_run.zip"
 
     @staticmethod
     def load_json_file(file: str) -> dict:
@@ -75,9 +75,9 @@ class IntegrationTestUtils:
         return file_exists and file_not_empty
 
     @staticmethod
-    def run_pipeline(component: str,
-                     start_date: str,
-                     **kwargs):
+    def run_unittest_pipeline(component: str,
+                              start_date: str,
+                              **kwargs):
 
         #  need EMAIL_CRED in the environment before we import Processor
         os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
@@ -105,3 +105,36 @@ class IntegrationTestUtils:
         except SystemExit:
             # we will eventually want to throw these to the calling class to be dealt with
             pass
+
+    @staticmethod
+    def run_external_pipeline(component: str,
+                              start_date: str,
+                              email_cred_path: str,
+                              **kwargs):
+
+        #  need EMAIL_CRED in the environment before we import Processor
+        os.environ["EMAIL_CRED"] = email_cred_path
+        from Processor import run_Process, set_log_level
+
+        args_dict: dict = {}
+
+        # note, possible to override these values in the kwargs loop below
+        args_dict['live'] = False  # what is this
+        args_dict['noupload'] = True  # need to work out where to upload the results too when a test
+        args_dict['start_date'] = start_date
+        args_dict['component'] = component
+        args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH]
+        args_dict['log_level'] = 'info'
+        args_dict['clearup'] = False  # what is this
+
+        for key, value in kwargs.items():
+            args_dict[key] = value
+
+        log_level = args_dict['log_level']
+        set_log_level(log_level)
+
+        try:
+            run_Process(args_dict)
+        except SystemExit:
+            # we will eventually want to throw these to the calling class to be dealt with
+            pass

tests/integration/test_advisory.py → tests/integration/partial/test_advisory.py

@@ -3,7 +3,7 @@ import glob
 import os
 import unittest
 
-from integration.integration_test_utils import IntegrationTestUtils
+from integration.partial.integration_test_utils import IntegrationTestUtils
 
 
 class TestAdvisory(unittest.TestCase):
@@ -46,7 +46,7 @@ class TestAdvisory(unittest.TestCase):
     @staticmethod
     def run_advisory_pipeline():
         component = 'Advisory'
-        IntegrationTestUtils.run_pipeline(component, TestAdvisory.TEST_START_DATE)
+        IntegrationTestUtils.run_unittest_pipeline(component, TestAdvisory.TEST_START_DATE)
 
 
     def test_standard_run_input_status_success(self):

tests/integration/test_deposition.py → tests/integration/partial/test_deposition.py

@@ -2,7 +2,7 @@ import copy
 import os
 import unittest
 
-from integration.integration_test_utils import IntegrationTestUtils
+from integration.partial.integration_test_utils import IntegrationTestUtils
 
 
 class TestDeposition(unittest.TestCase):
@@ -30,7 +30,8 @@ class TestDeposition(unittest.TestCase):
         run_dict['WorkspacePathout'] = TestDeposition.TEST_OUT_PATH
         run_dict['WorkspacePath'] = TestDeposition.TEST_OUT_PATH
         run_dict['ServerName'] = ''  # nothing, as local machine
-        full_server_path = os.path.abspath("../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/")
+        full_server_path = os.path.abspath(
+            "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/")
         run_dict['Deposition']['ServerPathTemplate'] = full_server_path
 
         IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
@@ -39,7 +40,7 @@ class TestDeposition(unittest.TestCase):
     @staticmethod
     def run_depo_pipeline():
         component = 'Deposition'
-        IntegrationTestUtils.run_pipeline(component, TestDeposition.TEST_START_DATE)
+        IntegrationTestUtils.run_unittest_pipeline(component, TestDeposition.TEST_START_DATE)
 
 
     def test_standard_run_input_status_success(self):

tests/integration/test_env_suit.py → tests/integration/partial/test_env_suit.py

@@ -2,7 +2,7 @@ import copy
 import os
 import unittest
 
-from integration.integration_test_utils import IntegrationTestUtils
+from integration.partial.integration_test_utils import IntegrationTestUtils
 
 
 class TestEnvSuit(unittest.TestCase):
@@ -33,7 +33,8 @@ class TestEnvSuit(unittest.TestCase):
         run_dict['Environment']['INPUT_PATH'] = TestEnvSuit.TEST_OUT_PATH
         run_dict['Environment']['OUTPUT_PATH'] = TestEnvSuit.TEST_OUT_PATH
         run_dict['ServerName'] = ''  # nothing, as local machine
-        full_server_path = os.path.abspath("../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/")
+        full_server_path = os.path.abspath(
+            "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/")
         run_dict['Environment']['ServerPathTemplate'] = full_server_path
 
         IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
@@ -42,7 +43,7 @@ class TestEnvSuit(unittest.TestCase):
     @staticmethod
     def run_env_pipeline():
         component = 'Environment'
-        IntegrationTestUtils.run_pipeline(component, TestEnvSuit.TEST_START_DATE)
+        IntegrationTestUtils.run_unittest_pipeline(component, TestEnvSuit.TEST_START_DATE)
 
 
     def test_standard_run_input_status_success(self):

tests/integration/test_epi.py → tests/integration/partial/test_epi.py

@@ -2,7 +2,7 @@ import copy
 import os
 import unittest
 
-from integration.integration_test_utils import IntegrationTestUtils
+from integration.partial.integration_test_utils import IntegrationTestUtils
 
 
 class TestEpi(unittest.TestCase):
@@ -46,7 +46,7 @@ class TestEpi(unittest.TestCase):
     @staticmethod
     def run_epi_pipeline():
         component = 'Epidemiology'
-        IntegrationTestUtils.run_pipeline(component, TestEpi.TEST_START_DATE)
+        IntegrationTestUtils.run_unittest_pipeline(component, TestEpi.TEST_START_DATE)
 
 
     def test_standard_run_input_status_success(self):

tests/integration/test_survey.py → tests/integration/partial/test_survey.py

@@ -2,7 +2,7 @@ import copy
 import os
 import unittest
 
-from integration.integration_test_utils import IntegrationTestUtils
+from integration.partial.integration_test_utils import IntegrationTestUtils
 
 
 class TestSurvey(unittest.TestCase):
@@ -42,7 +42,7 @@ class TestSurvey(unittest.TestCase):
     @staticmethod
     def run_survey_pipeline():
         component = 'Survey'
-        IntegrationTestUtils.run_pipeline(component, TestSurvey.TEST_START_DATE)
+        IntegrationTestUtils.run_unittest_pipeline(component, TestSurvey.TEST_START_DATE)
 
 
     def test_standard_run_input_status_success(self):