diff --git a/configs/docker/run/launchDockerRunnerIaas.sh~ b/configs/docker/run/launchDockerRunnerIaas.sh~ new file mode 100755 index 0000000000000000000000000000000000000000..87979f81f38b34cfd9c280249e7885e20d5cabbe --- /dev/null +++ b/configs/docker/run/launchDockerRunnerIaas.sh~ @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +docker run -it -v "/storage/app/EWS_prod/code:/storage/app/EWS_prod/code" \ + -v "/storage/app/EWS_prod/regions:/storage/app/EWS_prod/regions" \ + -v "/storage/app/EWS_prod/envs/credentials:/storage/app/EWS_prod/envs/credentials" \ + -v "/storage/sftp:/storage/sftp" \ + -v "/mnt/ewsmanager/RDS/webdata:/storage/moved" \ + -w "/storage/app/EWS_prod/code" \ + --name ews_runner \ + lb584/ews_coordinator + +echo "launched" diff --git a/configs/docker/run/runDockerEWS.sh~ b/configs/docker/run/runDockerEWS.sh~ new file mode 100755 index 0000000000000000000000000000000000000000..d63eec634582cbf0b85d9e8b33653f8032544b07 --- /dev/null +++ b/configs/docker/run/runDockerEWS.sh~ @@ -0,0 +1,57 @@ +#!/bin/bash +set -e + +upload='--noupload' # default +config='/storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json' # default +rundate=$(date '+%Y%m%d') # default today + +SHORT=p:,c:,s:,h +LONG=component:,config:,rundate::,help +OPTS=$(getopt -a -n weather --options $SHORT --longoptions $LONG -- "$@") + +eval set -- "$OPTS" + +while : +do + case "$1" in + -p | --component ) + component="$2" + shift 2 + ;; + -c | --config ) + config="$2" + shift 2 + ;; + -s | --rundate ) + rundate="$2" + shift 2 + ;; + -h | --help) + "This is a launch script, example usage: runDockerEWS.sh -p Deposition -c -c /<path>/config_EastAfrica_fc_live.json -s 20220808" + exit 2 + ;; + --) + shift; + break + ;; + *) + echo "Unexpected option: $1" + ;; + esac +done + + + +function run_coordinator() { + printf "component: %s\nconfig: %s\nrundate: %s\nupload: %s\n" $component $config $rundate $upload; + docker container ls; + docker exec -it \ + ews_runner \ + /storage/app/EWS_prod/code/coordinator/run_Processor.sh \ + -p $component \ + -c $config \ + -s $rundate \ + $upload +} + +run_coordinator diff --git a/configs/docker/run/runDockerEWS2.sh b/configs/docker/run/runDockerEWS2.sh new file mode 100755 index 0000000000000000000000000000000000000000..414ff69b0a6dcb3b7d6506ec8fe95e11a3754ffb --- /dev/null +++ b/configs/docker/run/runDockerEWS2.sh @@ -0,0 +1 @@ +echo "where does this go?" diff --git a/tests/integration/full/full_test_env_suit.py~ b/tests/integration/full/full_test_env_suit.py~ new file mode 100644 index 0000000000000000000000000000000000000000..58c58b9150c07079d43cd748b733f00e953b2b63 --- /dev/null +++ b/tests/integration/full/full_test_env_suit.py~ @@ -0,0 +1,129 @@ +import argparse +import copy +import glob +import os +import sys +import unittest + +from integration.partial.integration_test_utils import IntegrationTestUtils + + +class FullTestEnvSuit(unittest.TestCase): + + TEST_OUT_PATH: str = "NOT_SET" + TEST_START_DATE: str = 'NOT_SET' + TEST_JOB_DIR: str = "NOT_SET" + + @classmethod + def setUpClass(cls) -> None: + # FullTestEnvSuit.write_temp_run_config_file() + # FullTestEnvSuit.run_env_pipeline() + status_file_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR) + status_file = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR, "STATUS_SUCCESS") + #os.makedirs(status_file_path, exist_ok = True) + #os.mknod(status_file) + pass + + + @staticmethod + def write_temp_run_config_file(): + default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH + default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config) + run_dict: dict = copy.deepcopy(default_config_dict) + run_dict['WorkspacePathout'] = FullTestEnvSuit.TEST_OUT_PATH + run_dict['WorkspacePath'] = FullTestEnvSuit.TEST_OUT_PATH + run_dict['Environment']['WORK_PATH'] = FullTestEnvSuit.TEST_OUT_PATH + run_dict['Environment']['INPUT_PATH'] = FullTestEnvSuit.TEST_OUT_PATH + run_dict['Environment']['OUTPUT_PATH'] = FullTestEnvSuit.TEST_OUT_PATH + + IntegrationTestUtils.TEMP_CONFIG_FILE_PATH = FullTestEnvSuit.TEST_OUT_PATH + "temp_config.json" + IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH) + + + @staticmethod + def run_env_pipeline(): + component = 'Environment' + IntegrationTestUtils.run_external_pipeline(component, + FullTestEnvSuit.TEST_START_DATE, + IntegrationTestUtils.EMAIL_CRED_PATH) + + + def test_standard_run_input_status_success(self): + status_file_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR, "STATUS_SUCCESS") + print(f"looking for {status_file_path}") + success_file_exists: bool = os.path.isfile(status_file_path) + self.assertTrue(success_file_exists) + + + def test_standard_run_input_all_regions_ran(self): + """ + working on the assumption that if there are images for each region, it must have run through + (at least past the region iteration) + """ + + east_africa_image_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR, + "plotting", "eastafrica", "images", "Weekly", + "suitability_eastafrica_leaf_rust_total_*_map.png") + ethiopia_image_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR, + "plotting", "ethiopia", "images", "Weekly", + "suitability_ethiopia_leaf_rust_total_*_map.png") + + ea_file_exists: bool = len(glob.glob(east_africa_image_path)) > 0 + eth_file_exists: bool = len(glob.glob(ethiopia_image_path)) > 0 + self.assertTrue(ea_file_exists) + self.assertTrue(eth_file_exists) + + + def test_standard_run_all_input_csvs_produced(self): + east_africa_csv_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR, + "plotting", "eastafrica", "input_csvs", "*.csv") + ethiopia_csv_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR, + "plotting", "ethiopia", "input_csvs", "*.csv") + + ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_csv_path) + eth_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(ethiopia_csv_path) + self.assertEqual(18, ea_csv_count) + self.assertEqual(18, eth_csv_count) + + + def test_standard_run_all_images_produced(self): + east_africa_image_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR, + "plotting", "eastafrica", "images", "Weekly", "*.png") + ethiopia_image_path = os.path.join(FullTestEnvSuit.TEST_OUT_PATH, FullTestEnvSuit.TEST_JOB_DIR, + "plotting", "ethiopia", "images", "Weekly", "*.png") + + ea_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(east_africa_image_path) + eth_csv_count: int = IntegrationTestUtils.count_files_in_wildcard(ethiopia_image_path) + self.assertEqual(6, ea_csv_count) + self.assertEqual(6, eth_csv_count) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--config', required = True) + parser.add_argument('--outdir', required = True) + parser.add_argument('--email_cred', required = True) + parser.add_argument('--run_date_type', required = False) + parser.add_argument('--custom_run_date', required = False) + parser.add_argument('unittest_args', nargs='*') + + _args = parser.parse_args() + _config_file: str = _args.config + _outdir: str = _args.outdir + _email_cred_path: str = _args.email_cred + _run_date_type: str = _args.run_date_type + _custom_run_date: str = _args.custom_run_date + + IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH = _config_file + + FullTestEnvSuit.TEST_OUT_PATH = _outdir + IntegrationTestUtils.EMAIL_CRED_PATH = _email_cred_path + + FullTestEnvSuit.TEST_START_DATE = "20230227" + + FullTestEnvSuit.TEST_JOB_DIR = "ENVIRONMENT_2.0" \ + "_" + FullTestEnvSuit.TEST_START_DATE + + # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone) + sys.argv[1:] = _args.unittest_args + unittest.main() diff --git a/tests/integration/full/run_tests.sh~ b/tests/integration/full/run_tests.sh~ new file mode 100755 index 0000000000000000000000000000000000000000..227feccfb93552944428e1c651388bea069c0ef6 --- /dev/null +++ b/tests/integration/full/run_tests.sh~ @@ -0,0 +1,26 @@ + +#note this script only works if run in the project docker image, or an environment with the necessary paths and +# software installed + +source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/ + +PACKAGES_DIR="../../../../" # assumes run from the tests/integration/partial dir + +flagdir=$PACKAGES_DIR/flagdir +epimodel=$PACKAGES_DIR/epimodel +advisory=$PACKAGES_DIR/advisory_builder +met_processing=$PACKAGES_DIR/met_extractor_v2/met_data_extraction +met_processor=$PACKAGES_DIR/environmental_suitability/environmental_suitability +plotting=$PACKAGES_DIR/plotting/plotting +post_processing=$PACKAGES_DIR/post_processing/ews_postprocessing +source_gen=$PACKAGES_DIR/source_gen +coordinator=$PACKAGES_DIR/coordinator/coordinator +coordinator_tests=$PACKAGES_DIR/coordinator/tests + +export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests + +echo $PYTHONPATH + +python full_test_deposition.py --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json \ +--outdir /storage/app/EWS_prod/regions/EastAfrica/workspace/ \ +--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json \ No newline at end of file diff --git a/tests/integration/partial/example_suite/TestResults_unit.coordinator.example_test.ExampleTest_2023-02-02_18-38-27.html b/tests/integration/partial/example_suite/TestResults_unit.coordinator.example_test.ExampleTest_2023-02-02_18-38-27.html new file mode 100644 index 0000000000000000000000000000000000000000..a041e2afcc9600b8ca5c8d7b262cdd428b59df31 --- /dev/null +++ b/tests/integration/partial/example_suite/TestResults_unit.coordinator.example_test.ExampleTest_2023-02-02_18-38-27.html @@ -0,0 +1,68 @@ +<!DOCTYPE html> +<html> +<head> + <title>Unittest Results</title> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css" integrity="sha384-1q8mTJOASx8j1Au+a5WDVnPi2lkFfwwEAa8hDDdjZlpLegxhjVME1fgjWPGmkzs7" crossorigin="anonymous"> +</head> +<body> + <div class="container"> + <div class="row"> + <div class="col-xs-12"> + <h2 class="text-capitalize">Unittest Results</h2> + <p class='attribute'><strong>Start Time: </strong>2023-02-02 18:38:27</p> + <p class='attribute'><strong>Duration: </strong>0 ms</p> + <p class='attribute'><strong>Summary: </strong>Total: 1, Pass: 1</p> + </div> + </div> + <div class="row"> + <div class="col-xs-12 col-sm-10 col-md-10"> + <table class='table table-hover table-responsive'> + <thead> + <tr> + <th>unit.coordinator.example_test.ExampleTest</th> + <th>Status</th> + <th></th> + </tr> + </thead> + <tbody> + <tr class='success'> + <td class="col-xs-10">test_v1_dewperiod_expected_results1</td> + <td class="col-xs-1"> + <span class="label label-success" style="display:block;width:40px;">Pass</span> + </td> + <td class="col-xs-1"> + </td> + </tr> + <tr> + <td colspan="3"> + Total: 1, Pass: 1 -- Duration: 0 ms + </td> + </tr> + </tbody> + </table> + </div> + </div> + </div> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.4/jquery.min.js"></script> + <script type="text/javascript"> + $(document).ready(function(){ + $('td').on('click', '.btn', function(e){ + e.preventDefault(); + e.stopImmediatePropagation(); + var $this = $(this); + var $nextRow = $this.closest('tr').next('tr'); + $nextRow.slideToggle("fast"); + $this.text(function(i, text){ + if (text === 'View') { + return 'Hide'; + } else { + return 'View'; + }; + }); + }); + }); + </script> +</body> +</html \ No newline at end of file diff --git a/tests/integration/partial/example_test.py b/tests/integration/partial/example_test.py new file mode 100644 index 0000000000000000000000000000000000000000..b5bc92effc2c693df930b4f996cc5371a54b0cc7 --- /dev/null +++ b/tests/integration/partial/example_test.py @@ -0,0 +1,16 @@ +import unittest + + +class ExampleTest(unittest.TestCase): + + def setUp(self) -> None: + super().setUp() + + + def test_v1_dewperiod_expected_results1(self): + self.assertTrue(True) + + +if __name__ == '__main__': + print("hello") + unittest.main() diff --git a/tests/integration/partial/run_tests.sh~ b/tests/integration/partial/run_tests.sh~ new file mode 100755 index 0000000000000000000000000000000000000000..81cfad5c56f12b01d40165c127b4bd0f9cdbe285 --- /dev/null +++ b/tests/integration/partial/run_tests.sh~ @@ -0,0 +1,24 @@ + +#note this script only works if run in the project docker image, or an environment with the necessary paths and +# software installed + +source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/ + +PACKAGES_DIR="../../../../" # assumes run from the tests/integration/partial dir + +flagdir=$PACKAGES_DIR/flagdir +epimodel=$PACKAGES_DIR/epimodel +advisory=$PACKAGES_DIR/advisory_builder +met_processing=$PACKAGES_DIR/met_extractor_v2/met_data_extraction +met_processor=$PACKAGES_DIR/environmental_suitability/environmental_suitability +plotting=$PACKAGES_DIR/plotting/plotting +post_processing=$PACKAGES_DIR/post_processing/ews_postprocessing +source_gen=$PACKAGES_DIR/source_gen +coordinator=$PACKAGES_DIR/coordinator/coordinator +coordinator_tests=$PACKAGES_DIR/coordinator/tests + +export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests + +echo $PYTHONPATH + +python3 runreport.py diff --git a/tests/integration/partial/runreport.py b/tests/integration/partial/runreport.py new file mode 100644 index 0000000000000000000000000000000000000000..faa20343d9d6b771331b3c9c71510fce403d2e4e --- /dev/null +++ b/tests/integration/partial/runreport.py @@ -0,0 +1,11 @@ +from unittest import TestLoader, TestSuite +from HtmlTestRunner import HTMLTestRunner +from unit.coordinator.example_test import ExampleTest + +example_tests = TestLoader().loadTestsFromTestCase(ExampleTest) + +suite = TestSuite([example_tests]) + +runner = HTMLTestRunner(output='example_suite') + +runner.run(suite) diff --git a/tests/integration/partial/runreport.py~ b/tests/integration/partial/runreport.py~ new file mode 100644 index 0000000000000000000000000000000000000000..e674836c9d7b00c92444ea7ae18f9610e4e5794e --- /dev/null +++ b/tests/integration/partial/runreport.py~ @@ -0,0 +1,11 @@ +from unittest import TestLoader, TestSuite +from HtmlTestRunner import HTMLTestRunner +import unit.coordinator.example_test.ExampleTest + +example_tests = TestLoader().loadTestsFromTestCase(ExampleTest) + +suite = TestSuite([example_tests]) + +runner = HTMLTestRunner(output='example_suite') + +runner.run(suite) diff --git a/tests/unit/coordinator/.coverage b/tests/unit/coordinator/.coverage new file mode 100644 index 0000000000000000000000000000000000000000..83a6bedd12acaf9faa978b84c5d6f9ee1c527fbe Binary files /dev/null and b/tests/unit/coordinator/.coverage differ diff --git a/tests/unit/coordinator/run_tests.sh b/tests/unit/coordinator/run_tests.sh new file mode 100755 index 0000000000000000000000000000000000000000..d0980bf2993ee4da1b59b1f9500e29b58e7f5742 --- /dev/null +++ b/tests/unit/coordinator/run_tests.sh @@ -0,0 +1,28 @@ + +#note this script only works if run in the project docker image, or an environment with the necessary paths and +# software installed + +source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/ + +PACKAGES_DIR="../../../../" # assumes run from the tests/integration/partial dir + +flagdir=$PACKAGES_DIR/flagdir +epimodel=$PACKAGES_DIR/epimodel +advisory=$PACKAGES_DIR/advisory_builder +met_processing=$PACKAGES_DIR/met_extractor_v2/met_data_extraction +met_processor=$PACKAGES_DIR/environmental_suitability/environmental_suitability +plotting=$PACKAGES_DIR/plotting/plotting +post_processing=$PACKAGES_DIR/post_processing/ews_postprocessing +source_gen=$PACKAGES_DIR/source_gen +coordinator=$PACKAGES_DIR/coordinator/coordinator +coordinator_tests=$PACKAGES_DIR/coordinator/tests + +export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests + +echo $PYTHONPATH + +python3 -m unittest test_advisory.py +python3 -m unittest test_deposition.py +python3 -m unittest test_env_suit.py +python3 -m unittest test_epi.py +python3 -m unittest test_survey.py \ No newline at end of file