Merge branch 'master' of gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-coordinator

.gitlab-ci.yml

@@ -92,7 +92,7 @@ stages:
 #      - echo $PYTHONPATH
 
 half_fat_tests:
-  when: always
+  when: manual
   stage: half-fat
   extends: .half_fat_before_script
   image: lb584/ews_coordinator:latest
@@ -100,10 +100,6 @@ half_fat_tests:
   script:
     - cd $CI_PROJECT_DIR/tests/integration/partial/
     - python3 -m coverage run run_test_suite.py $CI_PROJECT_DIR/test_reports
-#    - python3 -m coverage run -m unittest test_advisory.py
-#    - python3 -m coverage run -m unittest test_deposition.py
-#    - python3 -m coverage run -m unittest test_env_suit.py
-#    - python3 -m coverage run -m unittest test_survey.py
     - python3 -m coverage report
     - python3 -m coverage html -d $CI_PROJECT_DIR/coverage
 #    - ls
@@ -144,8 +140,6 @@ full_fat_depo:
     - >
       cd $CI_PROJECT_DIR/tests/integration/full/
 
-      touch $OUTPUT_DIR/hello.txt
-
       python3 full_test_deposition.py 
       --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
       --outdir $OUTPUT_DIR
@@ -168,3 +162,69 @@ full_fat_depo:
       - $OUTPUT_DIR/*/log.txt
       - $OUTPUT_DIR/*.txt
     expire_in: 10 days
+
+full_fat_env_suit:
+  when: manual
+  stage: full-fat
+  extends: .half_fat_before_script
+  image: lb584/ews_coordinator:latest
+  tags: ["ewas-production"]
+  script:
+    - >
+      cd $CI_PROJECT_DIR/tests/integration/full/
+
+      python3 full_test_env_suit.py 
+      --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+      --outdir $OUTPUT_DIR
+      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
+      --run_date_type $FULL_FAT_RUN_DATE
+      --custom_run_date $CUSTOM_RUN_DATE
+
+  after_script:
+    - >
+      if [[ "$FULL_FAT_COPY_OUTPUT" == "true" ]];
+      then
+      date=$(date '+%Y-%m-%d-%H%M');
+      mkdir ${TEST_WORKSPACE_DIR}/integration/full/${date};
+      cp -r $OUTPUT_DIR/* ${TEST_WORKSPACE_DIR}/integration/full/${date};
+      fi
+
+  artifacts:
+    when: on_failure
+    paths:
+      - $OUTPUT_DIR/*/log.txt
+      - $OUTPUT_DIR/*.txt
+    expire_in: 10 days
+
+full_fat_survey:
+  when: manual
+  stage: full-fat
+  extends: .half_fat_before_script
+  image: lb584/ews_coordinator:latest
+  tags: ["ewas-production"]
+  script:
+    - >
+      cd $CI_PROJECT_DIR/tests/integration/full/
+
+      python3 full_test_survey.py 
+      --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+      --outdir $OUTPUT_DIR
+      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
+      --run_date_type $FULL_FAT_RUN_DATE
+      --custom_run_date $CUSTOM_RUN_DATE
+
+  after_script:
+    - >
+      if [[ "$FULL_FAT_COPY_OUTPUT" == "true" ]];
+      then
+      date=$(date '+%Y-%m-%d-%H%M');
+      mkdir ${TEST_WORKSPACE_DIR}/integration/full/${date};
+      cp -r $OUTPUT_DIR/* ${TEST_WORKSPACE_DIR}/integration/full/${date};
+      fi
+
+  artifacts:
+    when: on_failure
+    paths:
+      - $OUTPUT_DIR/*/log.txt
+      - $OUTPUT_DIR/*.txt
+    expire_in: 10 days

configs/docker/run/launchDockerRunnerIaas.sh

@@ -4,8 +4,8 @@ set -e
 docker run -it -v "/storage/app/EWS_prod/code:/storage/app/EWS_prod/code" \
   -v "/storage/app/EWS_prod/regions:/storage/app/EWS_prod/regions" \
   -v "/storage/app/EWS_prod/envs/credentials:/storage/app/EWS_prod/envs/credentials" \
-  -v "/storage/sftp:/storage/sftp" \
-  -v "/storage/webdata:/storage/webdata" \
+  -v "/mnt/RFS/production_backup/:/storage/sftp" \
+  -v "/mnt/ewsmanager/RDS/webdata:/storage/moved" \
   -w "/storage/app/EWS_prod/code" \
   --name ews_runner \
   lb584/ews_coordinator

configs/docker/run/runDockerEWS.sh

@@ -2,6 +2,7 @@
 set -e
 
 upload='--noupload' # default
+config='/storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json' # default
 rundate=$(date '+%Y%m%d') # default today
 
 SHORT=p:,c:,s:,h
@@ -39,10 +40,12 @@ do
   esac
 done
 
-printf "component: %s\nconfig: %s\nrundate: %s\nupload: %s\n" $component $config $rundate $upload
+
 
 function run_coordinator() {
-  docker exec -it \
+    printf "component: %s\nconfig: %s\nrundate: %s\nupload: %s\n" $component $config $rundate $upload;
+    docker container ls;
+    docker exec \
     ews_runner \
     /storage/app/EWS_prod/code/coordinator/run_Processor.sh \
     -p $component \

tests/integration/full/full_test_deposition.py

@@ -8,17 +8,17 @@ import unittest
 from integration.partial.integration_test_utils import IntegrationTestUtils
 
 
-class TestFullDeposition(unittest.TestCase):
+class FullTestDeposition(unittest.TestCase):
 
-    TEST_OUT_PATH: str = "not_set"
+    TEST_OUT_PATH: str = "NOT_SET"
     TEST_START_DATE: str = 'NOT_SET'
-    TEST_JOB_DIR: str = "DEPOSITION_" + TEST_START_DATE
+    TEST_JOB_DIR: str = "NOT_SET"
 
 
     @classmethod
     def setUpClass(cls) -> None:
-        TestFullDeposition.write_temp_run_config_file()
-        TestFullDeposition.run_depo_pipeline()
+        FullTestDeposition.write_temp_run_config_file()
+        FullTestDeposition.run_depo_pipeline()
         pass
 
 
@@ -27,10 +27,10 @@ class TestFullDeposition(unittest.TestCase):
         default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
         default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
         run_dict: dict = copy.deepcopy(default_config_dict)
-        run_dict['WorkspacePathout'] = TestFullDeposition.TEST_OUT_PATH
-        run_dict['WorkspacePath'] = TestFullDeposition.TEST_OUT_PATH
+        run_dict['WorkspacePathout'] = FullTestDeposition.TEST_OUT_PATH
+        run_dict['WorkspacePath'] = FullTestDeposition.TEST_OUT_PATH
 
-        IntegrationTestUtils.TEMP_CONFIG_FILE_PATH = TestFullDeposition.TEST_OUT_PATH + "temp_config.json"
+        IntegrationTestUtils.TEMP_CONFIG_FILE_PATH = FullTestDeposition.TEST_OUT_PATH + "temp_config.json"
         IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
 
 
@@ -38,12 +38,12 @@ class TestFullDeposition(unittest.TestCase):
     def run_depo_pipeline():
         component = 'Deposition'
         IntegrationTestUtils.run_external_pipeline(component,
-                                                   TestFullDeposition.TEST_START_DATE,
+                                                   FullTestDeposition.TEST_START_DATE,
                                                    IntegrationTestUtils.EMAIL_CRED_PATH)
 
 
     def test_standard_run_input_status_success(self):
-        status_file_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR, "STATUS_SUCCESS")
+        status_file_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR, "STATUS_SUCCESS")
         success_file_exists: bool = os.path.isfile(status_file_path)
         self.assertTrue(success_file_exists)
 
@@ -54,10 +54,10 @@ class TestFullDeposition(unittest.TestCase):
         (at least past the region iteration)
         """
 
-        east_africa_image_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR,
+        east_africa_image_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR,
                                               "plotting", "eastafrica", "images", "Weekly",
                                               "deposition_eastafrica_leaf_rust_total_*_map.png")
-        ethiopia_image_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR,
+        ethiopia_image_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR,
                                            "plotting", "ethiopia", "images", "Weekly",
                                            "deposition_ethiopia_leaf_rust_total_*_map.png")
 
@@ -68,21 +68,21 @@ class TestFullDeposition(unittest.TestCase):
 
 
     def test_standard_run_all_input_csvs_produced(self):
-        east_africa_csv_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR,
+        east_africa_csv_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR,
                                             "plotting", "eastafrica", "input_csvs", "*.csv")
-        ethiopia_csv_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR,
+        ethiopia_csv_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR,
                                          "plotting", "ethiopia", "input_csvs", "*.csv")
 
         ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_csv_path)
         eth_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(ethiopia_csv_path)
-        self.assertEqual(8, ea_csv_count)
+        self.assertEqual(9, ea_csv_count)
         self.assertEqual(9, eth_csv_count)
 
 
     def test_standard_run_all_images_produced(self):
-        east_africa_image_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR,
+        east_africa_image_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR,
                                               "plotting", "eastafrica", "images", "Weekly", "*.png")
-        ethiopia_image_path = os.path.join(TestFullDeposition.TEST_OUT_PATH, TestFullDeposition.TEST_JOB_DIR,
+        ethiopia_image_path = os.path.join(FullTestDeposition.TEST_OUT_PATH, FullTestDeposition.TEST_JOB_DIR,
                                            "plotting", "ethiopia", "images", "Weekly", "*.png")
 
         ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_image_path)
@@ -109,22 +109,13 @@ if __name__ == '__main__':
 
     IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH = _config_file
 
-    TestFullDeposition.TEST_OUT_PATH = _outdir
+    FullTestDeposition.TEST_OUT_PATH = _outdir
     IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path
 
-    if _run_date_type == "today":
-        print("today")
-    elif _run_date_type == "yesterday":
-        print("yesterday")
-    elif _run_date_type == "custom":
-        print(_custom_run_date)
-        TestFullDeposition.TEST_START_DATE = _custom_run_date
-        TestFullDeposition.TEST_JOB_DIR = "DEPOSITION_" + _custom_run_date
-    else:
-        print("default")
-        TestFullDeposition.TEST_START_DATE = "20230126"
-        TestFullDeposition.TEST_JOB_DIR = "DEPOSITION_" + TestFullDeposition.TEST_START_DATE
-
-    # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
+    FullTestDeposition.TEST_START_DATE = IntegrationTestUtils.generate_run_date(_run_date_type, _custom_run_date)
+
+    FullTestDeposition.TEST_JOB_DIR = "DEPOSITION_" + FullTestDeposition.TEST_START_DATE
+
+    #  Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
     sys.argv[1:] = _args.unittest_args
     unittest.main()

tests/integration/full/full_test_env_suit.py

+import argparse
+import copy
+import glob
+import os
+import sys
+import unittest
+
+from integration.partial.integration_test_utils import IntegrationTestUtils
+
+
+class FullTestEnvSuit(unittest.TestCase):
+
+    TEST_OUT_PATH: str = "NOT_SET"
+    TEST_START_DATE: str = 'NOT_SET'
+    TEST_JOB_DIR: str = "NOT_SET"
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        FullTestEnvSuit.write_temp_run_config_file()
+        FullTestEnvSuit.run_env_pipeline()
+        status_file_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR)
+        status_file = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR, "STATUS_SUCCESS")
+        #os.makedirs(status_file_path, exist_ok = True)
+        #os.mknod(status_file)
+        pass
+
+
+    @staticmethod
+    def write_temp_run_config_file():
+        default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
+        default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
+        run_dict: dict = copy.deepcopy(default_config_dict)
+        run_dict['WorkspacePathout'] = FullTestEnvSuit.TEST_OUT_PATH
+        run_dict['WorkspacePath'] = FullTestEnvSuit.TEST_OUT_PATH
+        run_dict['Environment']['WORK_PATH'] = FullTestEnvSuit.TEST_OUT_PATH
+        run_dict['Environment']['INPUT_PATH'] = FullTestEnvSuit.TEST_OUT_PATH
+        run_dict['Environment']['OUTPUT_PATH'] = FullTestEnvSuit.TEST_OUT_PATH
+
+        IntegrationTestUtils.TEMP_CONFIG_FILE_PATH = FullTestEnvSuit.TEST_OUT_PATH + "temp_config.json"
+        IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
+
+
+    @staticmethod
+    def run_env_pipeline():
+        component = 'Environment'
+        IntegrationTestUtils.run_external_pipeline(component,
+                                                   FullTestEnvSuit.TEST_START_DATE,
+                                                   IntegrationTestUtils.EMAIL_CRED_PATH)
+
+
+    def test_standard_run_input_status_success(self):
+        status_file_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR, "STATUS_SUCCESS")
+        print(f"looking for {status_file_path}")
+        success_file_exists: bool = os.path.isfile(status_file_path)
+        self.assertTrue(success_file_exists)
+
+
+    def test_standard_run_input_all_regions_ran(self):
+        """
+        working on the assumption that if there are images for each region, it must have run through
+        (at least past the region iteration)
+        """
+
+        east_africa_image_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR,
+                                              "plotting", "eastafrica", "images", "Weekly",
+                                              "suitability_eastafrica_leaf_rust_total_*_map.png")
+        ethiopia_image_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR,
+                                           "plotting", "ethiopia", "images", "Weekly",
+                                           "suitability_ethiopia_leaf_rust_total_*_map.png")
+
+        ea_file_exists: bool = len(glob.glob(east_africa_image_path)) > 0
+        eth_file_exists: bool = len(glob.glob(ethiopia_image_path)) > 0
+        self.assertTrue(ea_file_exists)
+        self.assertTrue(eth_file_exists)
+
+
+    def test_standard_run_all_input_csvs_produced(self):
+        east_africa_csv_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR,
+                                            "plotting", "eastafrica", "input_csvs", "*.csv")
+        ethiopia_csv_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR,
+                                         "plotting", "ethiopia", "input_csvs", "*.csv")
+
+        ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_csv_path)
+        eth_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(ethiopia_csv_path)
+        self.assertEqual(18, ea_csv_count)
+        self.assertEqual(18, eth_csv_count)
+
+
+    def test_standard_run_all_images_produced(self):
+        east_africa_image_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR,
+                                              "plotting", "eastafrica", "images", "Weekly", "*.png")
+        ethiopia_image_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR,
+                                           "plotting", "ethiopia", "images", "Weekly", "*.png")
+
+        ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_image_path)
+        eth_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(ethiopia_image_path)
+        self.assertEqual(6, ea_csv_count)
+        self.assertEqual(6, eth_csv_count)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--config', required = True)
+    parser.add_argument('--outdir', required = True)
+    parser.add_argument('--email_cred', required = True)
+    parser.add_argument('--run_date_type', required = False)
+    parser.add_argument('--custom_run_date', required = False)
+    parser.add_argument('unittest_args', nargs='*')
+
+    _args = parser.parse_args()
+    _config_file: str = _args.config
+    _outdir: str = _args.outdir
+    _email_cred_path: str = _args.email_cred
+    _run_date_type: str = _args.run_date_type
+    _custom_run_date: str = _args.custom_run_date
+
+    IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH = _config_file
+
+    FullTestEnvSuit.TEST_OUT_PATH = _outdir
+    IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path
+
+    # FullTestEnvSuit.TEST_START_DATE = "20230227"
+    FullTestEnvSuit.TEST_START_DATE = IntegrationTestUtils.generate_run_date(_run_date_type, _custom_run_date)
+
+    FullTestEnvSuit.TEST_JOB_DIR = "ENVIRONMENT_2.0" \
+                                   "_" + FullTestEnvSuit.TEST_START_DATE
+
+    #  Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
+    sys.argv[1:] = _args.unittest_args
+    unittest.main()

tests/integration/full/full_test_survey.py

@@ -9,9 +9,9 @@ from integration.partial.integration_test_utils import IntegrationTestUtils
 
 class FullTestSurvey(unittest.TestCase):
 
-    TEST_OUT_PATH: str = "not_set"
-    TEST_START_DATE: str = '20230126'
-    TEST_JOB_DIR: str = "SURVEYDATA_" + TEST_START_DATE
+    TEST_OUT_PATH: str = "NOT_SET"
+    TEST_START_DATE: str = 'NOT_SET'
+    TEST_JOB_DIR: str = "NOT_SET"
 
     @classmethod
     def setUpClass(cls) -> None:
@@ -33,9 +33,9 @@ class FullTestSurvey(unittest.TestCase):
         IntegrationTestUtils.TEMP_CONFIG_FILE_PATH = FullTestSurvey.TEST_OUT_PATH + "temp_config.json"
         IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH)
 
-    @staticmethod
-    def unpack_dependencies():
-        IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_SURVEY_FILE_PATH, FullTestSurvey.TEST_OUT_PATH)
+    # @staticmethod
+    # def unpack_dependencies():
+    #     IntegrationTestUtils.unpack_zip(IntegrationTestUtils.EXAMPLE_SURVEY_FILE_PATH, FullTestSurvey.TEST_OUT_PATH)
 
     @staticmethod
     def run_survey_pipeline():
@@ -64,18 +64,24 @@ if __name__ == '__main__':
     parser.add_argument('--config', required = True)
     parser.add_argument('--outdir', required = True)
     parser.add_argument('--email_cred', required = True)
+    parser.add_argument('--run_date_type', required = False)
+    parser.add_argument('--custom_run_date', required = False)
     parser.add_argument('unittest_args', nargs='*')
 
     _args = parser.parse_args()
     _config_file: str = _args.config
     _outdir: str = _args.outdir
     _email_cred_path: str = _args.email_cred
+    _run_date_type: str = _args.run_date_type
+    _custom_run_date: str = _args.custom_run_date
 
     IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH = _config_file
 
     FullTestSurvey.TEST_OUT_PATH = _outdir
     IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path
 
+    FullTestSurvey.TEST_START_DATE = IntegrationTestUtils.generate_run_date(_run_date_type, _custom_run_date)
+
     # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
     sys.argv[1:] = _args.unittest_args
     unittest.main()

tests/integration/full/run_tests.sh

@@ -21,6 +21,6 @@ export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_processing:$met_
 
 echo $PYTHONPATH
 
-python full_test_deposition.py --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json \
+python full_test_env_suit.py --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json \
 --outdir /storage/app/EWS_prod/regions/EastAfrica/workspace/ \
---email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
\ No newline at end of file
+--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json

tests/integration/partial/integration_test_utils.py

+import datetime
 import glob
 import json
 import os
-from datetime import datetime
 from importlib import reload
 from typing import List
 from zipfile import ZipFile
@@ -31,7 +31,7 @@ class IntegrationTestUtils:
 
     @staticmethod
     def get_now_string() -> str:
-        nowstring: str = datetime.today().strftime('%Y-%m-%d_%H%M%S')
+        nowstring: str = datetime.date.today().strftime('%Y-%m-%d_%H%M%S')
         return nowstring
 
 
@@ -141,3 +141,27 @@ class IntegrationTestUtils:
         except SystemExit:
             # we will eventually want to throw these to the calling class to be dealt with
             pass
+
+
+    @staticmethod
+    def generate_run_date(run_date_type: str,
+                          custom_run_date: str) -> str:
+
+        if run_date_type == "today":
+            print("today")
+            today = datetime.date.today()
+            today_string = today.strftime("%Y%m%d")
+            result = today_string
+        elif run_date_type == "yesterday":
+            print("yesterday")
+            today = datetime.date.today()
+            yesterday: datetime = today - datetime.timedelta(days = 1)
+            yesterday_string = yesterday.strftime("%Y%m%d")
+            result = yesterday_string
+        elif run_date_type == "custom":
+            result = custom_run_date
+        else:
+            print("default")
+            result = "20230126"
+
+        return result

tests/integration/partial/run_test_suite.py

@@ -6,11 +6,15 @@ from HtmlTestRunner import HTMLTestRunner
 
 from integration.partial.test_deposition import TestDeposition
 from integration.partial.test_env_suit import TestEnvSuit
+from integration.partial.test_survey import  TestSurvey
+from integration.partial.test_advisory import TestAdvisory
 
 
 def run_tests_and_report(output: str):
     tests: TestSuite = TestLoader().loadTestsFromTestCase(TestDeposition)
     tests.addTests(TestLoader().loadTestsFromTestCase(TestEnvSuit))
+    tests.addTests(TestLoader().loadTestsFromTestCase(TestSurvey))
+    tests.addTests(TestLoader().loadTestsFromTestCase(TestAdvisory))
     runner = HTMLTestRunner(output=output, combine_reports = True)
 
     runner.run(tests)