diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 358bf1897ce9f1bfc210d65484289c45df4872ab..46c168b624593ebf23061c424537f54ee6971580 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,7 +57,7 @@ test: - python3 -m coverage run -m unittest * - python -m coverage report - python -m coverage html -d $CI_PROJECT_DIR/coverage - - cd $CI_PROJECT_DIR/tests/integration + - cd $CI_PROJECT_DIR/tests/integration/partial/ - python3 -m coverage run -m unittest test_advisory.py - python3 -m coverage run -m unittest test_deposition.py - python3 -m coverage run -m unittest test_env_suit.py diff --git a/configs/docker/build/build.sh b/configs/docker/build/build.sh index 0d8837ed215414ee60e012aafd44995b8b3f3c5a..8c7e590ec931fe50670b0b5a8655dfa24373686b 100644 --- a/configs/docker/build/build.sh +++ b/configs/docker/build/build.sh @@ -1,3 +1,3 @@ cp ../../../conda-env-py3EWS-withbuilds.yml .; -sudo docker build --target base_image -t ews_coordinator .; +sudo docker build -t ews_coordinator .; rm -v conda-env-py3EWS-withbuilds.yml; diff --git a/tests/integration/full/__init__.py b/tests/integration/full/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/integration/partial/__init__.py b/tests/integration/partial/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/integration/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py similarity index 58% rename from tests/integration/integration_test_utils.py rename to tests/integration/partial/integration_test_utils.py index 1ecd042d0b9f460ed00531d146edfeb3bce6683a..4e383b2d2e5cf7e27e1f88e6c1159d1753e303d4 100644 --- a/tests/integration/integration_test_utils.py +++ b/tests/integration/partial/integration_test_utils.py @@ -8,12 +8,12 @@ from zipfile import ZipFile class IntegrationTestUtils: - EMAIL_CRED_PATH: str = "../test_data/test_deployment/envs/Cred_gmail.json" - DEFAULT_CONFIG_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json" - TEMP_CONFIG_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/workspace/temp_config.json" - EXAMPLE_SURVEY_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_survey_run.zip" - EXAMPLE_DEPO_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_depo_run.zip" - EXAMPLE_ENV_SUIT_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_env_suit_run.zip" + EMAIL_CRED_PATH: str = "../../test_data/test_deployment/envs/Cred_gmail.json" + DEFAULT_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json" + TEMP_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/workspace/temp_config.json" + EXAMPLE_SURVEY_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_survey_run.zip" + EXAMPLE_DEPO_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_depo_run.zip" + EXAMPLE_ENV_SUIT_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_env_suit_run.zip" @staticmethod def load_json_file(file: str) -> dict: @@ -75,9 +75,9 @@ class IntegrationTestUtils: return file_exists and file_not_empty @staticmethod - def run_pipeline(component: str, - start_date: str, - **kwargs): + def run_unittest_pipeline(component: str, + start_date: str, + **kwargs): # need EMAIL_CRED in the environment before we import Processor os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH @@ -105,3 +105,36 @@ class IntegrationTestUtils: except SystemExit: # we will eventually want to throw these to the calling class to be dealt with pass + + @staticmethod + def run_external_pipeline(component: str, + start_date: str, + email_cred_path: str, + **kwargs): + + # need EMAIL_CRED in the environment before we import Processor + os.environ["EMAIL_CRED"] = email_cred_path + from Processor import run_Process, set_log_level + + args_dict: dict = {} + + # note, possible to override these values in the kwargs loop below + args_dict['live'] = False # what is this + args_dict['noupload'] = True # need to work out where to upload the results too when a test + args_dict['start_date'] = start_date + args_dict['component'] = component + args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH] + args_dict['log_level'] = 'info' + args_dict['clearup'] = False # what is this + + for key, value in kwargs.items(): + args_dict[key] = value + + log_level = args_dict['log_level'] + set_log_level(log_level) + + try: + run_Process(args_dict) + except SystemExit: + # we will eventually want to throw these to the calling class to be dealt with + pass diff --git a/tests/integration/run_tests.sh b/tests/integration/partial/run_tests.sh similarity index 100% rename from tests/integration/run_tests.sh rename to tests/integration/partial/run_tests.sh diff --git a/tests/integration/test_advisory.py b/tests/integration/partial/test_advisory.py similarity index 95% rename from tests/integration/test_advisory.py rename to tests/integration/partial/test_advisory.py index da2fbbcb9ecc2b26fc95d5d918e69f940f3b42e1..657570235cf947b7b0853430f21b658fab3e0399 100644 --- a/tests/integration/test_advisory.py +++ b/tests/integration/partial/test_advisory.py @@ -3,7 +3,7 @@ import glob import os import unittest -from integration.integration_test_utils import IntegrationTestUtils +from integration.partial.integration_test_utils import IntegrationTestUtils class TestAdvisory(unittest.TestCase): @@ -46,7 +46,7 @@ class TestAdvisory(unittest.TestCase): @staticmethod def run_advisory_pipeline(): component = 'Advisory' - IntegrationTestUtils.run_pipeline(component, TestAdvisory.TEST_START_DATE) + IntegrationTestUtils.run_unittest_pipeline(component, TestAdvisory.TEST_START_DATE) def test_standard_run_input_status_success(self): diff --git a/tests/integration/test_deposition.py b/tests/integration/partial/test_deposition.py similarity index 93% rename from tests/integration/test_deposition.py rename to tests/integration/partial/test_deposition.py index a0be65baa27dfc7b782845810f3ca0d4efa5023b..9652cb70802a08cf8a5551d65a719651771aaf4b 100644 --- a/tests/integration/test_deposition.py +++ b/tests/integration/partial/test_deposition.py @@ -2,7 +2,7 @@ import copy import os import unittest -from integration.integration_test_utils import IntegrationTestUtils +from integration.partial.integration_test_utils import IntegrationTestUtils class TestDeposition(unittest.TestCase): @@ -30,7 +30,8 @@ class TestDeposition(unittest.TestCase): run_dict['WorkspacePathout'] = TestDeposition.TEST_OUT_PATH run_dict['WorkspacePath'] = TestDeposition.TEST_OUT_PATH run_dict['ServerName'] = '' # nothing, as local machine - full_server_path = os.path.abspath("../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/") + full_server_path = os.path.abspath( + "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/") run_dict['Deposition']['ServerPathTemplate'] = full_server_path IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH) @@ -39,7 +40,7 @@ class TestDeposition(unittest.TestCase): @staticmethod def run_depo_pipeline(): component = 'Deposition' - IntegrationTestUtils.run_pipeline(component, TestDeposition.TEST_START_DATE) + IntegrationTestUtils.run_unittest_pipeline(component, TestDeposition.TEST_START_DATE) def test_standard_run_input_status_success(self): diff --git a/tests/integration/test_env_suit.py b/tests/integration/partial/test_env_suit.py similarity index 93% rename from tests/integration/test_env_suit.py rename to tests/integration/partial/test_env_suit.py index 017a57fe32d46344f7d6d4750f66eb7f1edcfc62..05c13723e96ba660bd48ae067a86321925f8512d 100644 --- a/tests/integration/test_env_suit.py +++ b/tests/integration/partial/test_env_suit.py @@ -2,7 +2,7 @@ import copy import os import unittest -from integration.integration_test_utils import IntegrationTestUtils +from integration.partial.integration_test_utils import IntegrationTestUtils class TestEnvSuit(unittest.TestCase): @@ -33,7 +33,8 @@ class TestEnvSuit(unittest.TestCase): run_dict['Environment']['INPUT_PATH'] = TestEnvSuit.TEST_OUT_PATH run_dict['Environment']['OUTPUT_PATH'] = TestEnvSuit.TEST_OUT_PATH run_dict['ServerName'] = '' # nothing, as local machine - full_server_path = os.path.abspath("../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/") + full_server_path = os.path.abspath( + "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/") run_dict['Environment']['ServerPathTemplate'] = full_server_path IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH) @@ -42,7 +43,7 @@ class TestEnvSuit(unittest.TestCase): @staticmethod def run_env_pipeline(): component = 'Environment' - IntegrationTestUtils.run_pipeline(component, TestEnvSuit.TEST_START_DATE) + IntegrationTestUtils.run_unittest_pipeline(component, TestEnvSuit.TEST_START_DATE) def test_standard_run_input_status_success(self): diff --git a/tests/integration/test_epi.py b/tests/integration/partial/test_epi.py similarity index 96% rename from tests/integration/test_epi.py rename to tests/integration/partial/test_epi.py index fc8e8ac573310ccb5993b4f13fc2fda92c1b97c7..3f4f08c2bbad1561f3ec507ff3b7a14b3171f504 100644 --- a/tests/integration/test_epi.py +++ b/tests/integration/partial/test_epi.py @@ -2,7 +2,7 @@ import copy import os import unittest -from integration.integration_test_utils import IntegrationTestUtils +from integration.partial.integration_test_utils import IntegrationTestUtils class TestEpi(unittest.TestCase): @@ -46,7 +46,7 @@ class TestEpi(unittest.TestCase): @staticmethod def run_epi_pipeline(): component = 'Epidemiology' - IntegrationTestUtils.run_pipeline(component, TestEpi.TEST_START_DATE) + IntegrationTestUtils.run_unittest_pipeline(component, TestEpi.TEST_START_DATE) def test_standard_run_input_status_success(self): diff --git a/tests/integration/test_survey.py b/tests/integration/partial/test_survey.py similarity index 93% rename from tests/integration/test_survey.py rename to tests/integration/partial/test_survey.py index 00f12154180573d27daf93e72c57dd911132a1a7..7b4622336b1fa75e47e4fbeb6acdc05e9598e7f4 100644 --- a/tests/integration/test_survey.py +++ b/tests/integration/partial/test_survey.py @@ -2,7 +2,7 @@ import copy import os import unittest -from integration.integration_test_utils import IntegrationTestUtils +from integration.partial.integration_test_utils import IntegrationTestUtils class TestSurvey(unittest.TestCase): @@ -42,7 +42,7 @@ class TestSurvey(unittest.TestCase): @staticmethod def run_survey_pipeline(): component = 'Survey' - IntegrationTestUtils.run_pipeline(component, TestSurvey.TEST_START_DATE) + IntegrationTestUtils.run_unittest_pipeline(component, TestSurvey.TEST_START_DATE) def test_standard_run_input_status_success(self):