diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ee01a5e0dbfce161c573ac75bb737bce4225ef72..53a52e6ac20145666f8fda61788dff4189250d65 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -173,7 +173,7 @@ full_fat_advisory:
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
- --custom_dir_prefix "temp_advisory_$RUN_TIME"
+ --custom_dir_prefix "temp_advisory_2024-07-11_102855"
python3 full_test_deposition.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
@@ -183,7 +183,7 @@ full_fat_advisory:
--test_report_dir $CI_PROJECT_DIR/test_reports
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
- --custom_dir_prefix "temp_advisory_$RUN_TIME"
+ --custom_dir_prefix "temp_advisory_2024-07-11_102855"
python3 full_test_env_suit.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
@@ -192,7 +192,7 @@ full_fat_advisory:
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
- --custom_dir_prefix "temp_advisory_$RUN_TIME"
+ --custom_dir_prefix "temp_advisory_2024-07-11_102855"
python3 full_test_advisory.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
@@ -201,6 +201,7 @@ full_fat_advisory:
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
+ --custom_dir_prefix "temp_advisory_2024-07-11_102855"
full_fat_epi:
when: manual
diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py
index 48cadf3a44715a55c933d51947e847196090e53a..c1d00db19463a9a0f136bca9de039a9e7cedb0f1 100644
--- a/tests/integration/partial/integration_test_utils.py
+++ b/tests/integration/partial/integration_test_utils.py
@@ -131,7 +131,7 @@ class IntegrationTestUtils:
TEST_OUT_PATH, which is the top-level test directory for this run, not the TEST_JOB_DIR, which is the dir
for the pipeline being tested
"""
- runner = HTMLTestRunner(output = IntegrationTestUtils.TEST_OUT_PATH)
+ runner = HTMLTestRunner(output = IntegrationTestUtils.TEST_OUT_PATH, log = True, report_name = f"{test_prefix}_results")
result: TestResult = runner.run(tests)
return result.wasSuccessful()
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/delete_config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/delete_config_EastAfrica_fc_live.json
deleted file mode 100644
index 2df9315ac38efed24719b2f6b1e6523557889420..0000000000000000000000000000000000000000
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/delete_config_EastAfrica_fc_live.json
+++ /dev/null
@@ -1,358 +0,0 @@
-{
- "RegionName" : "EastAfrica",
- "SubRegionNames" : ["EastAfrica","Ethiopia"],
- "StartTime" : "?",
- "StartString" : "?",
- "WorkspacePathout" : "set_in_the_code",
- "WorkspacePath" : "set_in_the_code",
- "ResourcesPath" : "../../test_data/test_deployment/regions/EastAfrica/resources/",
- "ServerPath" : "OVERRIDDEN",
- "ServerName" : "OVERRIDDEN, set to empty string in tests",
- "ServerKey" : "not used, set to empty string in tests",
- "Survey" : {
- "ProcessPreJob" : "process_pre_job_survey",
- "ProcessInJob" : "process_in_job_survey",
- "ProcessEWSPlotting": "process_EWS_plotting_survey",
- "AcceptableDowntimeDays": 70,
- "SeasonStartString" : "20220930",
- "SkipServerDownload" : true,
- "ServerCredentialsFile" : "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/Cred-ODK-EIAR.json",
- "ServerPathExtra" : "OVERRIDDEN",
- "FormEdits" : {
- "wheat_rust_survey_1_0" : {
- "add" : {
- "Origin" : "ODK-server",
- "PublishedLevel" : "Raw"
-
- }
- },
- "akpyJHvYxkLKPkxFJnPyTW" : {
- "add" : {
- "Origin" : "kobo-server",
- "PublishedLevel" : "Raw"
- },
- "filter_by_list": {
- "surveyor_infromation-country" : ["Kenya", "Ethiopia"]
- }
- },
- "WRSIS" : {}
- },
- "Groups" : {
- "PROD" : {
- "Origin" : ["ODK-server", "kobo-server"]
- }
- },
- "GroupBy" : ["Origin"],
- "GroupsToIgnore" : ["ODK-server", "kobo-server", "newODK", "newODK2", "CSV-CAM"],
- "SurveyFormat" : "ODK",
- "SurveyorNameCol" : "surveyor_infromation-surveyor_name",
- "SourcesRegionName" : "EastAfrica",
- "SourcesConfigDefault" : "../../test_data/test_deployment/regions/EastAfrica/resources/source_gen/configs/config_EastAfrica_mapspam2017.json"
- },
- "Environment" : {
- "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/Ethiopia/fromMO/daily_name/",
- "InputFileTemplate" : "WR_EnvSuit_Met_Ethiopia_${StartString}_fc",
- "TimeExpectedAvailable" : "0800",
- "ProcessPreJob" : "process_pre_job_server_download",
- "ProcessInJob" : "process_in_job_env2_0",
- "ProcessEWSPlotting" : "process_EWS_plotting_env2_0",
- "RESOURCES_PATH": "../../test_data/test_deployment/regions/EastAfrica/resources/met_extractor/",
- "WORK_PATH" : "../../test_data/test_deployment/regions/EastAfrica/workspace/",
- "INPUT_PATH" : "../../test_data/test_deployment/regions/EastAfrica/workspace/",
- "OUTPUT_PATH" : "../../test_data/test_deployment/regions/EastAfrica/workspace/",
- "SYS_CONFIG" : "../../test_data/test_deployment/regions/EastAfrica/resources/met_extractor/configs/SYS_CONFIG_PINE.json",
- "FIELD_NAME_CONSTANTS" : "../../test_data/test_deployment/regions/EastAfrica/resources/met_extractor/configs/FIELD_NAME_CONSTANTS.csv",
- "RUN_TYPE" : "operational",
- "EXTRACTION_DAYS": 7,
- "FORECAST_DAYS": 6,
- "STRAINS": ["LeafRust","StemRust","StripeRust","LeafRust_TempOnly","StemRust_TempOnly","StripeRust_TempOnly"],
- "PARAMS": {
- "LeafRust": {
- "suitability_modules": ["semibool_dewperiod"],
- "past_steps": 0,
- "future_steps": 2,
- "thresholds": {
- "temperature": [2,15,20,30],
- "precipitation": 0,
- "relative_humidity": 90
- }
- },
- "StemRust": {
- "suitability_modules": ["semibool_dewperiod"],
- "past_steps": 0,
- "future_steps": 2,
- "thresholds": {
- "temperature": [2,15,24,30],
- "precipitation": 0,
- "relative_humidity": 90
- }
- },
- "StripeRust": {
- "suitability_modules": ["v1_dewperiod"],
- "past_steps": 0,
- "future_steps": 7,
- "thresholds": {
- "temperature": [2.37,19.8],
- "precipitation": 0,
- "relative_humidity": 90
- }
- },
- "LeafRust_TempOnly": {
- "suitability_modules": ["semibool_dewperiod"],
- "past_steps": 0,
- "future_steps": 2,
- "thresholds": {
- "temperature": [2,15,20,30],
- "precipitation": -1,
- "relative_humidity": 90
- }
- },
- "StemRust_TempOnly": {
- "suitability_modules": ["semibool_dewperiod"],
- "past_steps": 0,
- "future_steps": 2,
- "thresholds": {
- "temperature": [2,15,24,30],
- "precipitation": -1,
- "relative_humidity": 90
- }
- },
- "StripeRust_TempOnly": {
- "suitability_modules": ["v1_dewperiod"],
- "past_steps": 0,
- "future_steps": 7,
- "thresholds": {
- "temperature": [2.37,19.8],
- "precipitation": -1,
- "relative_humidity": 90
- }
- }
- },
- "EWS-Plotting" : {
- "SysConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/sys/SYS_CONFIG_PINE.json",
- "RunConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/env_suit/RUN_CONFIG_ENV.json",
- "EastAfrica" : {
- "ChartConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
- "FilterForCountry" : "False"
- },
- "Ethiopia" : {
- "ChartConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_ETHIOPIA_PINE.json",
- "FilterForCountry" : "True"
- }
- }
- },
- "Deposition" : {
- "ServerPathTemplate" : "OVERRIDDEN",
- "InputFileTemplate" : "WR_NAME_Ethiopia_${StartString}_fc",
- "DataFileTemplate" : "deposition_srcs_allregions_${StartString}.nc",
- "TimeExpectedAvailable" : "0800",
- "ProcessPreJob" : "process_pre_job_server_download",
- "ProcessInJob" : "process_in_job_dep",
- "ProcessEWSPlotting" : "process_EWS_plotting_dep",
- "EWS-Plotting" : {
- "SysConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/sys/SYS_CONFIG_PINE.json",
- "NameExtractionConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/name_extraction/NAME_EXTRACTION_CONFIG.json",
- "RunConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/deposition/RUN_CONFIG_DEPO.json",
- "RunConfigNorm" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/deposition/RUN_CONFIG_DEPO_NORMALIZED.json",
- "EastAfrica" : {
- "ChartConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
- "Normalize" : "False"
- },
- "Ethiopia" : {
- "ChartConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_ETHIOPIA_PINE.json",
- "Normalize" : "True"
- }
- }
- },
- "Epidemiology" : {
- "DiseaseNames" : ["StemRust"],
- "CalculationSpanDays" : [0,1],
- "TimeStep_hours": "3",
- "ProcessPreJob" : "process_pre_job_epi",
- "ProcessInJob" : "process_in_job_epi",
- "ProcessEWSPlotting" : "process_EWS_plotting_epi",
- "Host" : {
- "MaxFieldsPerCell" : "1",
- "TargetRaster" : "../../test_data/test_deployment/regions/EastAfrica/resources/epimodel/assets/wheat_area_frac_MapSPAM2010_EastAfrica_clipped.tif",
- "HostRasters" : {
- "201001010000" : "../../test_data/test_deployment/regions/EastAfrica/resources/epimodel/assets/wheat_area_frac_MapSPAM2010_EastAfrica_clipped.tif"
- }
- },
- "Deposition" : {
- "VariableNames" : ["P_GRAMINIS_DEPOSITION"],
- "PathTemplate" : "${WorkspacePath}DEPOSITION_${DateString}/WR_NAME_Ethiopia_${DateString}_fc/",
- "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS",
- "FileListerFunction" : "list_onefile_operational",
- "FileNameTemplate" : "deposition_srcs_allregions_${DateString}.nc",
- "FileNamePrepared" : "?"
- },
- "Environment" : {
- "PathTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${DateString}/processed/${RegionName}/${DiseaseName}/",
- "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS",
- "FileListerFunction" : "list_onefile_operational",
- "FileNameTemplate" : "RIE_value.nc",
- "FileNamePrepared" : "?"
- },
- "Env" : {
- "SuccessFileTemplate" : "${WorkspacePath}EPI_${YesterdayString}/STATUS_SUCCESS",
- "PathTemplate" : "${WorkspacePath}EPI_${YesterdayString}/${RegionName}/${DiseaseName}/",
- "FileNameTemplate" : "infections_config_EastAfrica_fc_live_Epidemiology_*_env_progression.csv",
- "FileListerFunction" : "list_onefile_historical",
- "FileLoaderFunction" : "load_and_restructure_epi_file",
- "FileNamePrepared" : "?"
- },
- "ps" : {
- "SuccessFileTemplate" : "${WorkspacePath}EPI_${YesterdayString}/STATUS_SUCCESS",
- "PathTemplate" : "${WorkspacePath}EPI_${YesterdayString}/${RegionName}/${DiseaseName}/",
- "FileNameTemplate" : "infections_config_EastAfrica_fc_live_Epidemiology_*_psbeta0.004gamma0.00025alpha1.0_progression.csv",
- "FileListerFunction" : "list_onefile_historical",
- "FileLoaderFunction" : "load_and_restructure_epi_file",
- "FileNamePrepared" : "?"
- },
- "Epi" : [
- {
- "model" : "Env",
- "modelArguments" : {},
- "infectionRasterFileName" : "?",
- "description": "env. suitability",
- "rescale_output_by_host_raster": false,
- "analysis" : {
- "vmin" : 0.0e+0,
- "vmax" : 1.5e+1,
- "subplot_position" : [0,0],
- "cmapString" : "CMRmap_r",
- "bounds" : [32.5,48.0,3.3,15.0],
- "UTMprojection" : 37
- }
- },{
- "model" : "ps",
- "modelArguments": {
- "beta": 4.0e-3,
- "gamma": 2.5e-4,
- "alpha": 1.0,
- "infection": "previous"
- },
- "infectionRasterFileName" : "?",
- "description": "SE[beta*log10(alpha*D)+gamma*P]",
- "rescale_output_by_host_raster": true,
- "analysis" : {
- "vmin" : 0.0e+0,
- "vmax" : 1,
- "subplot_position" : [0,1],
- "cmapString" : "CMRmap_r",
- "bounds": [32.5,48.0,3.3,15.0],
- "UTMprojection" : 37
- }
- }
- ],
- "EWS-Plotting" : {
- "SysConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/sys/SYS_CONFIG_PINE.json",
- "RunConfig_seasonsofar" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/epi/RUN_CONFIG_EPI.json",
- "RunConfig_seasonplusforecast" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/epi/RUN_CONFIG_EPI.json",
- "ChartConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_ETHIOPIA_PINE.json",
- "PlottingRegionName" : "Ethiopia",
- "EpiCase" : "ps"
- }
- },
- "Advisory" :
- {
- "ProcessPreJob" : "query_past_successes",
- "ProcessInJob" : "process_in_job_advisory",
- "ProcessEWSPlotting" : "do_nothing",
- "seasonStartString" : "20210901",
- "SurveyPathTemplate" : "${WorkspacePath}/SURVEYDATA_${dateString}/ExportCSV/Processed_SurveyData.csv",
- "DependentComponents" : [
- "Deposition",
- "Environment"
- ],
- "Environment" : {
- "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS",
- "DataPathTemplate" : "${WorkspacePath}/ENVIRONMENT_2.0_${dateString}/plotting/${SubRegionNameLower}/input_nc/",
- "PlotPathTemplate" : "${WorkspacePath}/ENVIRONMENT_2.0_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/",
- "PlotFilenameTemplates" : {
- "stripe" : "${envPlotPath}/suitability_${SubRegionNameLower}_stripe_rust_total_${dateString}0000_${EnvEndString}0000_map.png",
- "stem" : "${envPlotPath}/suitability_${SubRegionNameLower}_stem_rust_total_${dateString}0000_${EnvEndString}0000_map.png",
- "leaf" : "${envPlotPath}/suitability_${SubRegionNameLower}_leaf_rust_total_${dateString}0000_${EnvEndString}0000_map.png"
- },
- "Aggregation" : {
- "raster_path_template" : "${envDataPath}/suitability_${SubRegionNameLower}_${disease_name_snakecase}_total.nc",
- "raster_name_template" : "RIE_${disease_name_camelcase}_%",
- "shape_fn_entry" : "ShapeFilenameAdmin2",
- "shape_coln" : "ADM2_EN",
- "show_columns" : ["ADM0_EN","ADM1_EN"],
- "separate_rows_for_coln":"ADM0_EN",
- "func" : "mean",
- "multiply_by" : 100,
- "float_format" : ".0f",
- "display_format" : ".0f",
- "save_path_template" : "${jobPath}/images/suitability_${SubRegionNameLower}_${disease_name_snakecase}_mean_per_zone",
- "max_entries" : 20
- }
- },
- "Deposition" : {
- "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS",
- "DataPathTemplate" : "${WorkspacePath}/DEPOSITION_${dateString}/plotting/${SubRegionNameLower}/input_nc/",
- "PlotPathTemplate" : "${WorkspacePath}/DEPOSITION_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/",
- "PlotFilenameTemplates" : {
- "stripe": "${depPlotPath}/deposition_${SubRegionNameLower}_stripe_rust_total_${dateString}0000_${weekAheadString}0000_map.png",
- "stem": "${depPlotPath}/deposition_${SubRegionNameLower}_stem_rust_total_${dateString}0000_${weekAheadString}0000_map.png",
- "leaf": "${depPlotPath}/deposition_${SubRegionNameLower}_leaf_rust_total_${dateString}0000_${weekAheadString}0000_map.png"
- },
- "Aggregation" : {
- "raster_path_template" : "${depDataPath}/deposition_${SubRegionNameLower}_${disease_name_snakecase}_total.nc",
- "raster_name_template" : "spores_per_m2_per_week_${disease_name_camelcase}",
- "shape_fn_entry" : "ShapeFilenameAdmin2",
- "shape_coln" : "ADM2_EN",
- "show_columns" : ["ADM0_EN","ADM1_EN"],
- "separate_rows_for_coln":"ADM0_EN",
- "func" : "mean",
- "multiply_by" : 1,
- "float_format" : ".1e",
- "display_format" : ".0f",
- "save_path_template" : "${jobPath}/images/deposition_${SubRegionNameLower}_${disease_name_snakecase}_mean_per_zone",
- "max_entries" : 20
- }
- },
- "NOT_USING_Epidemiology_NOT_USING" : {
- "SuccessFileTemplate" : "${WorkspacePath}EPI_${StartString}/STATUS_SUCCESS",
- "DataPathTemplate" : "${WorkspacePath}/EPI_${dateString}/plotting/input_csvs/",
- "PlotPathTemplate" : "${WorkspacePath}/EPI_${dateString}/plotting/images/",
- "PlotFilenameTemplates" : {
- "stem": "${epiPlotPath}/infection_${SubRegionNameLower}_stem_seasonincforecast_*0300_*0000_map.png"
- },
- "Aggregation" : {
- "raster_path_template" : "${epiDataPath}/${SubRegionNameLower}_${disease_name_snakecase}_masked.nc",
- "raster_name_template" : "predicted_infected_area_${disease_name_camelcase}",
- "shape_fn_entry" : "ShapeFilenameAdmin2",
- "shape_coln" : "ADM2_EN",
- "show_columns" : ["ADM0_EN","ADM1_EN"],
- "separate_rows_for_coln":"ADM0_EN",
- "func" : "mean",
- "multiply_by" : 1,
- "float_format" : ".1e",
- "display_format" : ".0f",
- "save_path_template" : "${jobPath}/images/infection_${SubRegionNameLower}_${disease_name_snakecase}_mean_per_zone",
- "max_entries" : 20
- }
- },
- "Surveys" : {
- "variety_names_tidy_fn" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/configs/config_EastAfrica_variety_names_tidy.json",
- "CountryColumnName" : "surveyor_infromation-country",
- "EastAfrica" : {
- "CountryList" : ["Ethiopia", "Kenya", "Tanzania", "Zambia", "Uganda"],
- "ShapeFilenameAdmin1" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
- "ShapeFilenameAdmin2" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
- "ShapeFilenameToAggregate" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
- "Extent" : [21.5, 47, -18.5, 15],
- "LabelZoom" : 5
- },
- "Ethiopia" : {
- "ShapeFilenameAdmin1" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
- "ShapeFilenameAdmin2" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
- "Extent" : [32.8, 47, 3.2, 15],
- "LabelZoom" : 7
- }
- },
- "logos" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/logos_Ethiopia_plotting.png"
- }
-}