FAQ | This is a LIVE service | Changelog

Skip to content
Snippets Groups Projects
Commit 16787655 authored by L. Bower's avatar L. Bower
Browse files

more work to make full-fat tests wotk with new resources structure

parent f1085db5
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 4
− 3
View file @ 16787655
......@@ -173,7 +173,7 @@ full_fat_advisory:
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
--custom_dir_prefix "temp_advisory_$RUN_TIME"
--custom_dir_prefix "temp_advisory_2024-07-11_102855"
python3 full_test_deposition.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
......@@ -183,7 +183,7 @@ full_fat_advisory:
--test_report_dir $CI_PROJECT_DIR/test_reports
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
--custom_dir_prefix "temp_advisory_$RUN_TIME"
--custom_dir_prefix "temp_advisory_2024-07-11_102855"
python3 full_test_env_suit.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
......@@ -192,7 +192,7 @@ full_fat_advisory:
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
--custom_dir_prefix "temp_advisory_$RUN_TIME"
--custom_dir_prefix "temp_advisory_2024-07-11_102855"
python3 full_test_advisory.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
......@@ -201,6 +201,7 @@ full_fat_advisory:
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
--custom_dir_prefix "temp_advisory_2024-07-11_102855"
full_fat_epi:
when: manual
......
tests/integration/partial/integration_test_utils.py
+ 1
− 1
View file @ 16787655
......@@ -131,7 +131,7 @@ class IntegrationTestUtils:
TEST_OUT_PATH, which is the top-level test directory for this run, not the TEST_JOB_DIR, which is the dir
for the pipeline being tested
"""
runner = HTMLTestRunner(output = IntegrationTestUtils.TEST_OUT_PATH)
runner = HTMLTestRunner(output = IntegrationTestUtils.TEST_OUT_PATH, log = True, report_name = f"{test_prefix}_results")
result: TestResult = runner.run(tests)
return result.wasSuccessful()
......
tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/delete_config_EastAfrica_fc_live.json deleted 100644 → 0
+ 0
− 358
View file @ f1085db5
{
"RegionName" : "EastAfrica",
"SubRegionNames" : ["EastAfrica","Ethiopia"],
"StartTime" : "?",
"StartString" : "?",
"WorkspacePathout" : "set_in_the_code",
"WorkspacePath" : "set_in_the_code",
"ResourcesPath" : "../../test_data/test_deployment/regions/EastAfrica/resources/",
"ServerPath" : "OVERRIDDEN",
"ServerName" : "OVERRIDDEN, set to empty string in tests",
"ServerKey" : "not used, set to empty string in tests",
"Survey" : {
"ProcessPreJob" : "process_pre_job_survey",
"ProcessInJob" : "process_in_job_survey",
"ProcessEWSPlotting": "process_EWS_plotting_survey",
"AcceptableDowntimeDays": 70,
"SeasonStartString" : "20220930",
"SkipServerDownload" : true,
"ServerCredentialsFile" : "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/Cred-ODK-EIAR.json",
"ServerPathExtra" : "OVERRIDDEN",
"FormEdits" : {
"wheat_rust_survey_1_0" : {
"add" : {
"Origin" : "ODK-server",
"PublishedLevel" : "Raw"
}
},
"akpyJHvYxkLKPkxFJnPyTW" : {
"add" : {
"Origin" : "kobo-server",
"PublishedLevel" : "Raw"
},
"filter_by_list": {
"surveyor_infromation-country" : ["Kenya", "Ethiopia"]
}
},
"WRSIS" : {}
},
"Groups" : {
"PROD" : {
"Origin" : ["ODK-server", "kobo-server"]
}
},
"GroupBy" : ["Origin"],
"GroupsToIgnore" : ["ODK-server", "kobo-server", "newODK", "newODK2", "CSV-CAM"],
"SurveyFormat" : "ODK",
"SurveyorNameCol" : "surveyor_infromation-surveyor_name",
"SourcesRegionName" : "EastAfrica",
"SourcesConfigDefault" : "../../test_data/test_deployment/regions/EastAfrica/resources/source_gen/configs/config_EastAfrica_mapspam2017.json"
},
"Environment" : {
"ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/Ethiopia/fromMO/daily_name/",
"InputFileTemplate" : "WR_EnvSuit_Met_Ethiopia_${StartString}_fc",
"TimeExpectedAvailable" : "0800",
"ProcessPreJob" : "process_pre_job_server_download",
"ProcessInJob" : "process_in_job_env2_0",
"ProcessEWSPlotting" : "process_EWS_plotting_env2_0",
"RESOURCES_PATH": "../../test_data/test_deployment/regions/EastAfrica/resources/met_extractor/",
"WORK_PATH" : "../../test_data/test_deployment/regions/EastAfrica/workspace/",
"INPUT_PATH" : "../../test_data/test_deployment/regions/EastAfrica/workspace/",
"OUTPUT_PATH" : "../../test_data/test_deployment/regions/EastAfrica/workspace/",
"SYS_CONFIG" : "../../test_data/test_deployment/regions/EastAfrica/resources/met_extractor/configs/SYS_CONFIG_PINE.json",
"FIELD_NAME_CONSTANTS" : "../../test_data/test_deployment/regions/EastAfrica/resources/met_extractor/configs/FIELD_NAME_CONSTANTS.csv",
"RUN_TYPE" : "operational",
"EXTRACTION_DAYS": 7,
"FORECAST_DAYS": 6,
"STRAINS": ["LeafRust","StemRust","StripeRust","LeafRust_TempOnly","StemRust_TempOnly","StripeRust_TempOnly"],
"PARAMS": {
"LeafRust": {
"suitability_modules": ["semibool_dewperiod"],
"past_steps": 0,
"future_steps": 2,
"thresholds": {
"temperature": [2,15,20,30],
"precipitation": 0,
"relative_humidity": 90
}
},
"StemRust": {
"suitability_modules": ["semibool_dewperiod"],
"past_steps": 0,
"future_steps": 2,
"thresholds": {
"temperature": [2,15,24,30],
"precipitation": 0,
"relative_humidity": 90
}
},
"StripeRust": {
"suitability_modules": ["v1_dewperiod"],
"past_steps": 0,
"future_steps": 7,
"thresholds": {
"temperature": [2.37,19.8],
"precipitation": 0,
"relative_humidity": 90
}
},
"LeafRust_TempOnly": {
"suitability_modules": ["semibool_dewperiod"],
"past_steps": 0,
"future_steps": 2,
"thresholds": {
"temperature": [2,15,20,30],
"precipitation": -1,
"relative_humidity": 90
}
},
"StemRust_TempOnly": {
"suitability_modules": ["semibool_dewperiod"],
"past_steps": 0,
"future_steps": 2,
"thresholds": {
"temperature": [2,15,24,30],
"precipitation": -1,
"relative_humidity": 90
}
},
"StripeRust_TempOnly": {
"suitability_modules": ["v1_dewperiod"],
"past_steps": 0,
"future_steps": 7,
"thresholds": {
"temperature": [2.37,19.8],
"precipitation": -1,
"relative_humidity": 90
}
}
},
"EWS-Plotting" : {
"SysConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/sys/SYS_CONFIG_PINE.json",
"RunConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/env_suit/RUN_CONFIG_ENV.json",
"EastAfrica" : {
"ChartConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
"FilterForCountry" : "False"
},
"Ethiopia" : {
"ChartConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_ETHIOPIA_PINE.json",
"FilterForCountry" : "True"
}
}
},
"Deposition" : {
"ServerPathTemplate" : "OVERRIDDEN",
"InputFileTemplate" : "WR_NAME_Ethiopia_${StartString}_fc",
"DataFileTemplate" : "deposition_srcs_allregions_${StartString}.nc",
"TimeExpectedAvailable" : "0800",
"ProcessPreJob" : "process_pre_job_server_download",
"ProcessInJob" : "process_in_job_dep",
"ProcessEWSPlotting" : "process_EWS_plotting_dep",
"EWS-Plotting" : {
"SysConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/sys/SYS_CONFIG_PINE.json",
"NameExtractionConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/name_extraction/NAME_EXTRACTION_CONFIG.json",
"RunConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/deposition/RUN_CONFIG_DEPO.json",
"RunConfigNorm" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/deposition/RUN_CONFIG_DEPO_NORMALIZED.json",
"EastAfrica" : {
"ChartConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
"Normalize" : "False"
},
"Ethiopia" : {
"ChartConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_ETHIOPIA_PINE.json",
"Normalize" : "True"
}
}
},
"Epidemiology" : {
"DiseaseNames" : ["StemRust"],
"CalculationSpanDays" : [0,1],
"TimeStep_hours": "3",
"ProcessPreJob" : "process_pre_job_epi",
"ProcessInJob" : "process_in_job_epi",
"ProcessEWSPlotting" : "process_EWS_plotting_epi",
"Host" : {
"MaxFieldsPerCell" : "1",
"TargetRaster" : "../../test_data/test_deployment/regions/EastAfrica/resources/epimodel/assets/wheat_area_frac_MapSPAM2010_EastAfrica_clipped.tif",
"HostRasters" : {
"201001010000" : "../../test_data/test_deployment/regions/EastAfrica/resources/epimodel/assets/wheat_area_frac_MapSPAM2010_EastAfrica_clipped.tif"
}
},
"Deposition" : {
"VariableNames" : ["P_GRAMINIS_DEPOSITION"],
"PathTemplate" : "${WorkspacePath}DEPOSITION_${DateString}/WR_NAME_Ethiopia_${DateString}_fc/",
"SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS",
"FileListerFunction" : "list_onefile_operational",
"FileNameTemplate" : "deposition_srcs_allregions_${DateString}.nc",
"FileNamePrepared" : "?"
},
"Environment" : {
"PathTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${DateString}/processed/${RegionName}/${DiseaseName}/",
"SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS",
"FileListerFunction" : "list_onefile_operational",
"FileNameTemplate" : "RIE_value.nc",
"FileNamePrepared" : "?"
},
"Env" : {
"SuccessFileTemplate" : "${WorkspacePath}EPI_${YesterdayString}/STATUS_SUCCESS",
"PathTemplate" : "${WorkspacePath}EPI_${YesterdayString}/${RegionName}/${DiseaseName}/",
"FileNameTemplate" : "infections_config_EastAfrica_fc_live_Epidemiology_*_env_progression.csv",
"FileListerFunction" : "list_onefile_historical",
"FileLoaderFunction" : "load_and_restructure_epi_file",
"FileNamePrepared" : "?"
},
"ps" : {
"SuccessFileTemplate" : "${WorkspacePath}EPI_${YesterdayString}/STATUS_SUCCESS",
"PathTemplate" : "${WorkspacePath}EPI_${YesterdayString}/${RegionName}/${DiseaseName}/",
"FileNameTemplate" : "infections_config_EastAfrica_fc_live_Epidemiology_*_psbeta0.004gamma0.00025alpha1.0_progression.csv",
"FileListerFunction" : "list_onefile_historical",
"FileLoaderFunction" : "load_and_restructure_epi_file",
"FileNamePrepared" : "?"
},
"Epi" : [
{
"model" : "Env",
"modelArguments" : {},
"infectionRasterFileName" : "?",
"description": "env. suitability",
"rescale_output_by_host_raster": false,
"analysis" : {
"vmin" : 0.0e+0,
"vmax" : 1.5e+1,
"subplot_position" : [0,0],
"cmapString" : "CMRmap_r",
"bounds" : [32.5,48.0,3.3,15.0],
"UTMprojection" : 37
}
},{
"model" : "ps",
"modelArguments": {
"beta": 4.0e-3,
"gamma": 2.5e-4,
"alpha": 1.0,
"infection": "previous"
},
"infectionRasterFileName" : "?",
"description": "SE[beta*log10(alpha*D)+gamma*P]",
"rescale_output_by_host_raster": true,
"analysis" : {
"vmin" : 0.0e+0,
"vmax" : 1,
"subplot_position" : [0,1],
"cmapString" : "CMRmap_r",
"bounds": [32.5,48.0,3.3,15.0],
"UTMprojection" : 37
}
}
],
"EWS-Plotting" : {
"SysConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/sys/SYS_CONFIG_PINE.json",
"RunConfig_seasonsofar" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/epi/RUN_CONFIG_EPI.json",
"RunConfig_seasonplusforecast" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/epi/RUN_CONFIG_EPI.json",
"ChartConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_ETHIOPIA_PINE.json",
"PlottingRegionName" : "Ethiopia",
"EpiCase" : "ps"
}
},
"Advisory" :
{
"ProcessPreJob" : "query_past_successes",
"ProcessInJob" : "process_in_job_advisory",
"ProcessEWSPlotting" : "do_nothing",
"seasonStartString" : "20210901",
"SurveyPathTemplate" : "${WorkspacePath}/SURVEYDATA_${dateString}/ExportCSV/Processed_SurveyData.csv",
"DependentComponents" : [
"Deposition",
"Environment"
],
"Environment" : {
"SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS",
"DataPathTemplate" : "${WorkspacePath}/ENVIRONMENT_2.0_${dateString}/plotting/${SubRegionNameLower}/input_nc/",
"PlotPathTemplate" : "${WorkspacePath}/ENVIRONMENT_2.0_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/",
"PlotFilenameTemplates" : {
"stripe" : "${envPlotPath}/suitability_${SubRegionNameLower}_stripe_rust_total_${dateString}0000_${EnvEndString}0000_map.png",
"stem" : "${envPlotPath}/suitability_${SubRegionNameLower}_stem_rust_total_${dateString}0000_${EnvEndString}0000_map.png",
"leaf" : "${envPlotPath}/suitability_${SubRegionNameLower}_leaf_rust_total_${dateString}0000_${EnvEndString}0000_map.png"
},
"Aggregation" : {
"raster_path_template" : "${envDataPath}/suitability_${SubRegionNameLower}_${disease_name_snakecase}_total.nc",
"raster_name_template" : "RIE_${disease_name_camelcase}_%",
"shape_fn_entry" : "ShapeFilenameAdmin2",
"shape_coln" : "ADM2_EN",
"show_columns" : ["ADM0_EN","ADM1_EN"],
"separate_rows_for_coln":"ADM0_EN",
"func" : "mean",
"multiply_by" : 100,
"float_format" : ".0f",
"display_format" : ".0f",
"save_path_template" : "${jobPath}/images/suitability_${SubRegionNameLower}_${disease_name_snakecase}_mean_per_zone",
"max_entries" : 20
}
},
"Deposition" : {
"SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS",
"DataPathTemplate" : "${WorkspacePath}/DEPOSITION_${dateString}/plotting/${SubRegionNameLower}/input_nc/",
"PlotPathTemplate" : "${WorkspacePath}/DEPOSITION_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/",
"PlotFilenameTemplates" : {
"stripe": "${depPlotPath}/deposition_${SubRegionNameLower}_stripe_rust_total_${dateString}0000_${weekAheadString}0000_map.png",
"stem": "${depPlotPath}/deposition_${SubRegionNameLower}_stem_rust_total_${dateString}0000_${weekAheadString}0000_map.png",
"leaf": "${depPlotPath}/deposition_${SubRegionNameLower}_leaf_rust_total_${dateString}0000_${weekAheadString}0000_map.png"
},
"Aggregation" : {
"raster_path_template" : "${depDataPath}/deposition_${SubRegionNameLower}_${disease_name_snakecase}_total.nc",
"raster_name_template" : "spores_per_m2_per_week_${disease_name_camelcase}",
"shape_fn_entry" : "ShapeFilenameAdmin2",
"shape_coln" : "ADM2_EN",
"show_columns" : ["ADM0_EN","ADM1_EN"],
"separate_rows_for_coln":"ADM0_EN",
"func" : "mean",
"multiply_by" : 1,
"float_format" : ".1e",
"display_format" : ".0f",
"save_path_template" : "${jobPath}/images/deposition_${SubRegionNameLower}_${disease_name_snakecase}_mean_per_zone",
"max_entries" : 20
}
},
"NOT_USING_Epidemiology_NOT_USING" : {
"SuccessFileTemplate" : "${WorkspacePath}EPI_${StartString}/STATUS_SUCCESS",
"DataPathTemplate" : "${WorkspacePath}/EPI_${dateString}/plotting/input_csvs/",
"PlotPathTemplate" : "${WorkspacePath}/EPI_${dateString}/plotting/images/",
"PlotFilenameTemplates" : {
"stem": "${epiPlotPath}/infection_${SubRegionNameLower}_stem_seasonincforecast_*0300_*0000_map.png"
},
"Aggregation" : {
"raster_path_template" : "${epiDataPath}/${SubRegionNameLower}_${disease_name_snakecase}_masked.nc",
"raster_name_template" : "predicted_infected_area_${disease_name_camelcase}",
"shape_fn_entry" : "ShapeFilenameAdmin2",
"shape_coln" : "ADM2_EN",
"show_columns" : ["ADM0_EN","ADM1_EN"],
"separate_rows_for_coln":"ADM0_EN",
"func" : "mean",
"multiply_by" : 1,
"float_format" : ".1e",
"display_format" : ".0f",
"save_path_template" : "${jobPath}/images/infection_${SubRegionNameLower}_${disease_name_snakecase}_mean_per_zone",
"max_entries" : 20
}
},
"Surveys" : {
"variety_names_tidy_fn" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/configs/config_EastAfrica_variety_names_tidy.json",
"CountryColumnName" : "surveyor_infromation-country",
"EastAfrica" : {
"CountryList" : ["Ethiopia", "Kenya", "Tanzania", "Zambia", "Uganda"],
"ShapeFilenameAdmin1" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
"ShapeFilenameAdmin2" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
"ShapeFilenameToAggregate" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
"Extent" : [21.5, 47, -18.5, 15],
"LabelZoom" : 5
},
"Ethiopia" : {
"ShapeFilenameAdmin1" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
"ShapeFilenameAdmin2" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
"Extent" : [32.8, 47, 3.2, 15],
"LabelZoom" : 7
}
},
"logos" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/logos_Ethiopia_plotting.png"
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment