From 60fcd3562199c5b162257d990d60ee31a918c58c Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Fri, 27 Jan 2023 14:04:49 +0000
Subject: [PATCH] repackaging integration tests
---
.gitlab-ci.yml | 2 +-
configs/docker/build/build.sh | 2 +-
tests/integration/full/__init__.py | 0
tests/integration/partial/__init__.py | 0
.../{ => partial}/integration_test_utils.py | 51 +++++++++++++++----
tests/integration/{ => partial}/run_tests.sh | 0
.../{ => partial}/test_advisory.py | 4 +-
.../{ => partial}/test_deposition.py | 7 +--
.../{ => partial}/test_env_suit.py | 7 +--
tests/integration/{ => partial}/test_epi.py | 4 +-
.../integration/{ => partial}/test_survey.py | 4 +-
11 files changed, 58 insertions(+), 23 deletions(-)
create mode 100644 tests/integration/full/__init__.py
create mode 100644 tests/integration/partial/__init__.py
rename tests/integration/{ => partial}/integration_test_utils.py (58%)
rename tests/integration/{ => partial}/run_tests.sh (100%)
rename tests/integration/{ => partial}/test_advisory.py (95%)
rename tests/integration/{ => partial}/test_deposition.py (93%)
rename tests/integration/{ => partial}/test_env_suit.py (93%)
rename tests/integration/{ => partial}/test_epi.py (96%)
rename tests/integration/{ => partial}/test_survey.py (93%)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 358bf18..46c168b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -57,7 +57,7 @@ test:
- python3 -m coverage run -m unittest *
- python -m coverage report
- python -m coverage html -d $CI_PROJECT_DIR/coverage
- - cd $CI_PROJECT_DIR/tests/integration
+ - cd $CI_PROJECT_DIR/tests/integration/partial/
- python3 -m coverage run -m unittest test_advisory.py
- python3 -m coverage run -m unittest test_deposition.py
- python3 -m coverage run -m unittest test_env_suit.py
diff --git a/configs/docker/build/build.sh b/configs/docker/build/build.sh
index 0d8837e..8c7e590 100644
--- a/configs/docker/build/build.sh
+++ b/configs/docker/build/build.sh
@@ -1,3 +1,3 @@
cp ../../../conda-env-py3EWS-withbuilds.yml .;
-sudo docker build --target base_image -t ews_coordinator .;
+sudo docker build -t ews_coordinator .;
rm -v conda-env-py3EWS-withbuilds.yml;
diff --git a/tests/integration/full/__init__.py b/tests/integration/full/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/integration/partial/__init__.py b/tests/integration/partial/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/integration/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py
similarity index 58%
rename from tests/integration/integration_test_utils.py
rename to tests/integration/partial/integration_test_utils.py
index 1ecd042..4e383b2 100644
--- a/tests/integration/integration_test_utils.py
+++ b/tests/integration/partial/integration_test_utils.py
@@ -8,12 +8,12 @@ from zipfile import ZipFile
class IntegrationTestUtils:
- EMAIL_CRED_PATH: str = "../test_data/test_deployment/envs/Cred_gmail.json"
- DEFAULT_CONFIG_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json"
- TEMP_CONFIG_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/workspace/temp_config.json"
- EXAMPLE_SURVEY_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_survey_run.zip"
- EXAMPLE_DEPO_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_depo_run.zip"
- EXAMPLE_ENV_SUIT_FILE_PATH: str = "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_env_suit_run.zip"
+ EMAIL_CRED_PATH: str = "../../test_data/test_deployment/envs/Cred_gmail.json"
+ DEFAULT_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json"
+ TEMP_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/workspace/temp_config.json"
+ EXAMPLE_SURVEY_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_survey_run.zip"
+ EXAMPLE_DEPO_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_depo_run.zip"
+ EXAMPLE_ENV_SUIT_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/example_env_suit_run.zip"
@staticmethod
def load_json_file(file: str) -> dict:
@@ -75,9 +75,9 @@ class IntegrationTestUtils:
return file_exists and file_not_empty
@staticmethod
- def run_pipeline(component: str,
- start_date: str,
- **kwargs):
+ def run_unittest_pipeline(component: str,
+ start_date: str,
+ **kwargs):
# need EMAIL_CRED in the environment before we import Processor
os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
@@ -105,3 +105,36 @@ class IntegrationTestUtils:
except SystemExit:
# we will eventually want to throw these to the calling class to be dealt with
pass
+
+ @staticmethod
+ def run_external_pipeline(component: str,
+ start_date: str,
+ email_cred_path: str,
+ **kwargs):
+
+ # need EMAIL_CRED in the environment before we import Processor
+ os.environ["EMAIL_CRED"] = email_cred_path
+ from Processor import run_Process, set_log_level
+
+ args_dict: dict = {}
+
+ # note, possible to override these values in the kwargs loop below
+ args_dict['live'] = False # what is this
+ args_dict['noupload'] = True # need to work out where to upload the results too when a test
+ args_dict['start_date'] = start_date
+ args_dict['component'] = component
+ args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH]
+ args_dict['log_level'] = 'info'
+ args_dict['clearup'] = False # what is this
+
+ for key, value in kwargs.items():
+ args_dict[key] = value
+
+ log_level = args_dict['log_level']
+ set_log_level(log_level)
+
+ try:
+ run_Process(args_dict)
+ except SystemExit:
+ # we will eventually want to throw these to the calling class to be dealt with
+ pass
diff --git a/tests/integration/run_tests.sh b/tests/integration/partial/run_tests.sh
similarity index 100%
rename from tests/integration/run_tests.sh
rename to tests/integration/partial/run_tests.sh
diff --git a/tests/integration/test_advisory.py b/tests/integration/partial/test_advisory.py
similarity index 95%
rename from tests/integration/test_advisory.py
rename to tests/integration/partial/test_advisory.py
index da2fbbc..6575702 100644
--- a/tests/integration/test_advisory.py
+++ b/tests/integration/partial/test_advisory.py
@@ -3,7 +3,7 @@ import glob
import os
import unittest
-from integration.integration_test_utils import IntegrationTestUtils
+from integration.partial.integration_test_utils import IntegrationTestUtils
class TestAdvisory(unittest.TestCase):
@@ -46,7 +46,7 @@ class TestAdvisory(unittest.TestCase):
@staticmethod
def run_advisory_pipeline():
component = 'Advisory'
- IntegrationTestUtils.run_pipeline(component, TestAdvisory.TEST_START_DATE)
+ IntegrationTestUtils.run_unittest_pipeline(component, TestAdvisory.TEST_START_DATE)
def test_standard_run_input_status_success(self):
diff --git a/tests/integration/test_deposition.py b/tests/integration/partial/test_deposition.py
similarity index 93%
rename from tests/integration/test_deposition.py
rename to tests/integration/partial/test_deposition.py
index a0be65b..9652cb7 100644
--- a/tests/integration/test_deposition.py
+++ b/tests/integration/partial/test_deposition.py
@@ -2,7 +2,7 @@ import copy
import os
import unittest
-from integration.integration_test_utils import IntegrationTestUtils
+from integration.partial.integration_test_utils import IntegrationTestUtils
class TestDeposition(unittest.TestCase):
@@ -30,7 +30,8 @@ class TestDeposition(unittest.TestCase):
run_dict['WorkspacePathout'] = TestDeposition.TEST_OUT_PATH
run_dict['WorkspacePath'] = TestDeposition.TEST_OUT_PATH
run_dict['ServerName'] = '' # nothing, as local machine
- full_server_path = os.path.abspath("../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/")
+ full_server_path = os.path.abspath(
+ "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/")
run_dict['Deposition']['ServerPathTemplate'] = full_server_path
IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
@@ -39,7 +40,7 @@ class TestDeposition(unittest.TestCase):
@staticmethod
def run_depo_pipeline():
component = 'Deposition'
- IntegrationTestUtils.run_pipeline(component, TestDeposition.TEST_START_DATE)
+ IntegrationTestUtils.run_unittest_pipeline(component, TestDeposition.TEST_START_DATE)
def test_standard_run_input_status_success(self):
diff --git a/tests/integration/test_env_suit.py b/tests/integration/partial/test_env_suit.py
similarity index 93%
rename from tests/integration/test_env_suit.py
rename to tests/integration/partial/test_env_suit.py
index 017a57f..05c1372 100644
--- a/tests/integration/test_env_suit.py
+++ b/tests/integration/partial/test_env_suit.py
@@ -2,7 +2,7 @@ import copy
import os
import unittest
-from integration.integration_test_utils import IntegrationTestUtils
+from integration.partial.integration_test_utils import IntegrationTestUtils
class TestEnvSuit(unittest.TestCase):
@@ -33,7 +33,8 @@ class TestEnvSuit(unittest.TestCase):
run_dict['Environment']['INPUT_PATH'] = TestEnvSuit.TEST_OUT_PATH
run_dict['Environment']['OUTPUT_PATH'] = TestEnvSuit.TEST_OUT_PATH
run_dict['ServerName'] = '' # nothing, as local machine
- full_server_path = os.path.abspath("../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/")
+ full_server_path = os.path.abspath(
+ "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/")
run_dict['Environment']['ServerPathTemplate'] = full_server_path
IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
@@ -42,7 +43,7 @@ class TestEnvSuit(unittest.TestCase):
@staticmethod
def run_env_pipeline():
component = 'Environment'
- IntegrationTestUtils.run_pipeline(component, TestEnvSuit.TEST_START_DATE)
+ IntegrationTestUtils.run_unittest_pipeline(component, TestEnvSuit.TEST_START_DATE)
def test_standard_run_input_status_success(self):
diff --git a/tests/integration/test_epi.py b/tests/integration/partial/test_epi.py
similarity index 96%
rename from tests/integration/test_epi.py
rename to tests/integration/partial/test_epi.py
index fc8e8ac..3f4f08c 100644
--- a/tests/integration/test_epi.py
+++ b/tests/integration/partial/test_epi.py
@@ -2,7 +2,7 @@ import copy
import os
import unittest
-from integration.integration_test_utils import IntegrationTestUtils
+from integration.partial.integration_test_utils import IntegrationTestUtils
class TestEpi(unittest.TestCase):
@@ -46,7 +46,7 @@ class TestEpi(unittest.TestCase):
@staticmethod
def run_epi_pipeline():
component = 'Epidemiology'
- IntegrationTestUtils.run_pipeline(component, TestEpi.TEST_START_DATE)
+ IntegrationTestUtils.run_unittest_pipeline(component, TestEpi.TEST_START_DATE)
def test_standard_run_input_status_success(self):
diff --git a/tests/integration/test_survey.py b/tests/integration/partial/test_survey.py
similarity index 93%
rename from tests/integration/test_survey.py
rename to tests/integration/partial/test_survey.py
index 00f1215..7b46223 100644
--- a/tests/integration/test_survey.py
+++ b/tests/integration/partial/test_survey.py
@@ -2,7 +2,7 @@ import copy
import os
import unittest
-from integration.integration_test_utils import IntegrationTestUtils
+from integration.partial.integration_test_utils import IntegrationTestUtils
class TestSurvey(unittest.TestCase):
@@ -42,7 +42,7 @@ class TestSurvey(unittest.TestCase):
@staticmethod
def run_survey_pipeline():
component = 'Survey'
- IntegrationTestUtils.run_pipeline(component, TestSurvey.TEST_START_DATE)
+ IntegrationTestUtils.run_unittest_pipeline(component, TestSurvey.TEST_START_DATE)
def test_standard_run_input_status_success(self):
--
GitLab