diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 66565b5f9d55e7526645ccef35c8b6476c594a69..b15f4c494098e929e90bc388d9ebc7b2f761230e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -35,7 +35,7 @@ stages:
- half-fat
- full-fat
-.half_fat_before_script:
+.before_script:
before_script:
- echo "packages dir - " $PACKAGES_DIR
@@ -94,7 +94,7 @@ stages:
half_fat_tests:
when: manual
stage: half-fat
- extends: .half_fat_before_script
+ extends: .before_script
image: lb584/ews_coordinator:latest
script:
@@ -115,7 +115,7 @@ half_fat_tests:
half_fat_epi_tests:
when: manual
stage: half-fat
- extends: .half_fat_before_script
+ extends: .before_script
image: lb584/ews_coordinator:latest
script:
@@ -133,7 +133,7 @@ half_fat_epi_tests:
full_fat_depo:
when: manual
stage: full-fat
- extends: .half_fat_before_script
+ extends: .before_script
image: lb584/ews_coordinator:latest
tags: ["ewas-production"]
script:
@@ -166,7 +166,7 @@ full_fat_depo:
full_fat_env_suit:
when: manual
stage: full-fat
- extends: .half_fat_before_script
+ extends: .before_script
image: lb584/ews_coordinator:latest
tags: ["ewas-production"]
script:
@@ -199,7 +199,7 @@ full_fat_env_suit:
full_fat_survey:
when: manual
stage: full-fat
- extends: .half_fat_before_script
+ extends: .before_script
image: lb584/ews_coordinator:latest
tags: ["ewas-production"]
script:
@@ -228,3 +228,36 @@ full_fat_survey:
- $OUTPUT_DIR/*/log.txt
- $OUTPUT_DIR/*.txt
expire_in: 10 days
+
+full_fat_advisory:
+ when: manual
+ stage: full-fat
+ extends: .before_script
+ image: lb584/ews_coordinator:latest
+ tags: ["ewas-production"]
+ script:
+ - >
+ cd $CI_PROJECT_DIR/tests/integration/full/
+
+ python3 full_test_advisory.py
+ --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+ --outdir $OUTPUT_DIR
+ --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
+ --run_date_type $FULL_FAT_RUN_DATE
+ --custom_run_date $CUSTOM_RUN_DATE
+
+ after_script:
+ - >
+ if [[ "$FULL_FAT_COPY_OUTPUT" == "true" ]];
+ then
+ date=$(date '+%Y-%m-%d-%H%M');
+ mkdir ${TEST_WORKSPACE_DIR}/integration/full/${date};
+ cp -r $OUTPUT_DIR/* ${TEST_WORKSPACE_DIR}/integration/full/${date};
+ fi
+
+ artifacts:
+ when: on_failure
+ paths:
+ - $OUTPUT_DIR/*/log.txt
+ - $OUTPUT_DIR/*.txt
+ expire_in: 10 days
diff --git a/tests/integration/full/full_test_advisory.py b/tests/integration/full/full_test_advisory.py
new file mode 100644
index 0000000000000000000000000000000000000000..5a758b334c5868d3b2c94360a3522592f5feca52
--- /dev/null
+++ b/tests/integration/full/full_test_advisory.py
@@ -0,0 +1,104 @@
+import argparse
+import copy
+import os
+import sys
+import unittest
+
+from integration.partial.integration_test_utils import IntegrationTestUtils
+from integration.test_suites.advisory_test_suite import BaseAdvisoryTestSuite
+
+
+class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
+
+ def set_expected_values(self):
+ self.EXPECTED_EA_IMAGE_COUNT = 3
+ self.EXPECTED_ETH_IMAGE_COUNT = 3
+ self.EXPECTED_EA_SHAPE_COUNT = 6
+
+
+ def setUp(self) -> None:
+
+ self.set_expected_values()
+
+ if BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH is None or not os.path.isdir(BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH):
+ # if True:
+ FullTestAdvisory.write_temp_run_config_files()
+ FullTestAdvisory.run_dependent_pipelines()
+ FullTestAdvisory.run_advisory_pipeline()
+ else:
+ print(f"output in {BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH} already written, skipping rerun")
+
+
+ @staticmethod
+ def write_temp_run_config_files():
+ default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
+ default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
+ run_dict: dict = copy.deepcopy(default_config_dict)
+ run_dict['WorkspacePathout'] = BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH
+ run_dict['WorkspacePath'] = BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH
+ # we need to run the env suit pipeline as well
+ run_dict['Environment']['WORK_PATH'] = BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH
+ run_dict['Environment']['INPUT_PATH'] = BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH
+ run_dict['Environment']['OUTPUT_PATH'] = BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH
+ #make sure we activate server download
+ run_dict['Survey']['SkipServerDownload'] = False
+ survey_cred_file: str = run_dict['Survey']['ServerCredentialsFile']
+ run_dict['Survey']['ServerCredentialsFile'] = survey_cred_file.replace("Cred-ODK-EIAR.json", "Cred-ODK-EIAR-NO-WRSIS.json")
+ run_dict['Survey']['FormEdits'].pop('WRSIS', None)
+
+ # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
+ os.makedirs(BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH, exist_ok = True)
+ IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
+
+
+ @staticmethod
+ def run_dependent_pipelines():
+ IntegrationTestUtils.run_external_pipeline("Deposition",
+ BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_START_DATE,
+ IntegrationTestUtils.EMAIL_CRED_PATH)
+ IntegrationTestUtils.run_external_pipeline("Environment",
+ BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_START_DATE,
+ IntegrationTestUtils.EMAIL_CRED_PATH)
+ IntegrationTestUtils.run_external_pipeline("Survey",
+ "20230125",
+ IntegrationTestUtils.EMAIL_CRED_PATH)
+ pass
+
+ @staticmethod
+ def run_advisory_pipeline():
+ component = 'Advisory'
+ IntegrationTestUtils.run_unittest_pipeline(component, FullTestAdvisory.TEST_START_DATE)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--config', required = True)
+ parser.add_argument('--outdir', required = True)
+ parser.add_argument('--email_cred', required = True)
+ parser.add_argument('--run_date_type', required = False)
+ parser.add_argument('--custom_run_date', required = False)
+ parser.add_argument('unittest_args', nargs='*')
+
+ _args = parser.parse_args()
+ _config_file: str = _args.config
+ _outdir: str = _args.outdir
+ _email_cred_path: str = _args.email_cred
+ _run_date_type: str = _args.run_date_type
+ _custom_run_date: str = _args.custom_run_date
+
+ IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH = _config_file
+
+ nowstring: str = IntegrationTestUtils.get_now_string()
+ # prefix: str = "temp_advisory_" + nowstring
+ prefix: str = "temp_advisory"
+ BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH = _outdir + prefix + os.sep
+ IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path
+
+ BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_START_DATE = IntegrationTestUtils.generate_run_date(_run_date_type, _custom_run_date)
+
+ BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_JOB_DIR = "ADVISORY_" + BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_START_DATE
+
+ # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
+ sys.argv[1:] = _args.unittest_args
+ unittest.main()
diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py
index 6aef1ba3b7f81ffd0d0596364a5af21b664ac5f0..c039aea462f7e7e064f6d06c6deb1dd1e40b47ec 100644
--- a/tests/integration/partial/integration_test_utils.py
+++ b/tests/integration/partial/integration_test_utils.py
@@ -12,7 +12,7 @@ class IntegrationTestUtils:
EMAIL_CRED_PATH: str = "../../test_data/test_deployment/envs/Cred_gmail.json"
DEFAULT_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json"
TEST_WORKSPACE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/workspace/"
- TEMP_CONFIG_FILE_NAME: str = "temp_config.json"
+ TEMP_CONFIG_FILE_NAME: str = None
TEST_ASSETS_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/"
EXAMPLE_SURVEY_FILE_PATH: str = TEST_ASSETS_PATH + "example_survey_run.zip"
@@ -120,18 +120,21 @@ class IntegrationTestUtils:
# need EMAIL_CRED in the environment before we import Processor
os.environ["EMAIL_CRED"] = email_cred_path
+
+ import Processor
+ reload(Processor)
from Processor import run_Process, set_log_level
args_dict: dict = {}
# note, possible to override these values in the kwargs loop below
- args_dict['live'] = False # what is this
- args_dict['noupload'] = True # need to work out where to upload the results too when a test
+ args_dict['live'] = False
+ args_dict['noupload'] = True
args_dict['start_date'] = start_date
args_dict['component'] = component
args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_NAME]
args_dict['log_level'] = 'info'
- args_dict['clearup'] = False # what is this
+ args_dict['clearup'] = True
for key, value in kwargs.items():
args_dict[key] = value
diff --git a/tests/integration/partial/test_advisory.py b/tests/integration/partial/test_advisory.py
index 417ff2d2e1cacd5e600bea47f16e37d968c5c68d..c559f04da2f3a74a8565962f49794189e469bb76 100644
--- a/tests/integration/partial/test_advisory.py
+++ b/tests/integration/partial/test_advisory.py
@@ -11,6 +11,9 @@ class TestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
def set_expected_values(self):
BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_START_DATE = '20221001'
BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_JOB_DIR = "ADVISORY_" + BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_START_DATE
+ self.EXPECTED_EA_IMAGE_COUNT = 3
+ self.EXPECTED_ETH_IMAGE_COUNT = 3
+ self.EXPECTED_EA_SHAPE_COUNT = 3
def setUp(self) -> None:
diff --git a/tests/integration/test_suites/README.txt b/tests/integration/test_suites/README.txt
new file mode 100644
index 0000000000000000000000000000000000000000..99226cd302ee1a49ed0d4e26716c20c079885f4f
--- /dev/null
+++ b/tests/integration/test_suites/README.txt
@@ -0,0 +1,3 @@
+to prevent the full integrations tests that extend the partial integrations tests running both sets of tests,
+the test suite is nested in a base class, to keep it at a separate package-level
+https://stackoverflow.com/questions/1323455/python-unit-test-with-base-and-sub-class
\ No newline at end of file
diff --git a/tests/integration/test_suites/advisory_test_suite.py b/tests/integration/test_suites/advisory_test_suite.py
index 6a9f4ddbcd173c05b21c95fd4e87f7821dc3d395..03b3f6cebb7ea96a38a8819eee687fe7f72f052b 100644
--- a/tests/integration/test_suites/advisory_test_suite.py
+++ b/tests/integration/test_suites/advisory_test_suite.py
@@ -16,7 +16,9 @@ class BaseAdvisoryTestSuite:
@abc.abstractmethod
def set_expected_values(self):
- pass
+ self.EXPECTED_EA_IMAGE_COUNT = 0
+ self.EXPECTED_ETH_IMAGE_COUNT = 0
+ self.EXPECTED_EA_SHAPE_COUNT = 0
def test_standard_run_input_status_success(self):
@@ -26,17 +28,17 @@ class BaseAdvisoryTestSuite:
def test_standard_run_input_all_docs_produced(self):
- east_africa_image_path = os.path.join(BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH,
+ east_africa_image_wildcard = os.path.join(BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH,
BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_JOB_DIR,
"tight-layout",
- "wheat_rust_advisory_template_EastAfrica_20221001.docx")
- ethiopia_image_path = os.path.join(BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH,
+ "wheat_rust_advisory_template_EastAfrica_*.docx")
+ ethiopia_image_wildcard = os.path.join(BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_OUT_PATH,
BaseAdvisoryTestSuite.AdvisoryTestSuite.TEST_JOB_DIR,
"tight-layout",
- "wheat_rust_advisory_template_Ethiopia_20221001.docx")
+ "wheat_rust_advisory_template_Ethiopia_*.docx")
- ea_file_exists: bool = os.path.isfile(east_africa_image_path)
- eth_file_exists: bool = os.path.isfile(ethiopia_image_path)
+ ea_file_exists: bool = len(glob.glob(east_africa_image_wildcard)) == 1
+ eth_file_exists: bool = len(glob.glob(ethiopia_image_wildcard)) == 1
self.assertTrue(ea_file_exists)
self.assertTrue(eth_file_exists)
@@ -53,8 +55,8 @@ class BaseAdvisoryTestSuite:
africa_image_count: int = len(glob.glob(east_africa_image_wildcard))
ethiopia_image_count: int = len(glob.glob(ethiopia_image_wildcard))
- self.assertEqual(3, africa_image_count)
- self.assertEqual(3, ethiopia_image_count)
+ self.assertEqual(self.EXPECTED_EA_IMAGE_COUNT, africa_image_count)
+ self.assertEqual(self.EXPECTED_ETH_IMAGE_COUNT, ethiopia_image_count)
def test_standard_run_input_shapefiles_produced(self):
@@ -63,5 +65,5 @@ class BaseAdvisoryTestSuite:
"images",
"*eastafrica*.shp")
- africa_image_count: int = len(glob.glob(east_africa_image_wildcard))
- self.assertEqual(3, africa_image_count)
\ No newline at end of file
+ africa_shape_count: int = len(glob.glob(east_africa_image_wildcard))
+ self.assertEqual(self.EXPECTED_EA_SHAPE_COUNT, africa_shape_count)
\ No newline at end of file