diff --git a/coordinator/ProcessorEpidemiology.py b/coordinator/ProcessorEpidemiology.py index 76daa9f56b1a3b0e409ca93a178e957d86008726..baadb590e6458624560aacd4abc5cd25b1f5f1de 100644 --- a/coordinator/ProcessorEpidemiology.py +++ b/coordinator/ProcessorEpidemiology.py @@ -8,6 +8,7 @@ import logging from pathlib import Path import os import shutil +from typing import List from numpy import argmax, unique from pandas import read_csv, DataFrame, to_datetime @@ -75,7 +76,7 @@ def calc_epi_date_range(init_str,span_days=[0,6]): return start_date, end_date -def process_pre_job_epi(input_args): +def process_pre_job_epi(input_args: dict): '''Returns a boolean as to whether the job is ready for full processing.''' logger.info('started process_pre_job_epi()') @@ -83,7 +84,7 @@ def process_pre_job_epi(input_args): # check pre-requisite jobs are complete query_past_successes(input_args) - config_fns = input_args.config_paths + config_fns: List[str] = input_args['config_paths'] for configFile in config_fns: @@ -91,7 +92,7 @@ def process_pre_job_epi(input_args): config_i = open_and_check_config(configFile) #determine end time, from config file - arg_start_date = input_args.start_date + arg_start_date: str = input_args['start_date'] calc_span_days = config_i['Epidemiology']['CalculationSpanDays'] assert len(calc_span_days) == 2 diff --git a/coordinator/ProcessorSurveys.py b/coordinator/ProcessorSurveys.py index c6ea0ea783281f44f8979237a6833088531d7c8d..064c40ab3c4aa7f5001b93701ca92fbc4bc52f18 100644 --- a/coordinator/ProcessorSurveys.py +++ b/coordinator/ProcessorSurveys.py @@ -48,37 +48,41 @@ def get_ODK_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat ODK_jar = form_credentials['ODK_jar'] assert os.path.exists(ODK_jar) - ODK_download = ['java', - '-jar', ODK_jar, - '--pull_aggregate', - '--form_id', form_credentials['form_id'], - '--storage_directory', ODK_output_path, - '--odk_url', form_credentials['url'], - '--odk_username',form_credentials['user'], - '--odk_password',form_credentials['pass']] + skip_download: bool = config['Survey'].get('SkipServerDownload', False) ODK_download_success = True - logger.debug('Performing ' + description_long) + if not skip_download: + try: + ODK_download = ['java', + '-jar', ODK_jar, + '--pull_aggregate', + '--form_id', form_credentials['form_id'], + '--storage_directory', ODK_output_path, + '--odk_url', form_credentials['url'], + '--odk_username', form_credentials['user'], + '--odk_password', form_credentials['pass']] - try: - # perform a pull from the ODK server, and if it fails write a warning message + logger.debug('Performing ' + description_long) - subprocess_and_log(ODK_download,description_short,description_long,log_type='warning',check=True) + # perform a pull from the ODK server, and if it fails write a warning message - except subprocess.CalledProcessError as e: - status.reset('WARNING') - ODK_download_success = False + subprocess_and_log(ODK_download,description_short,description_long,log_type='warning',check=True) + + except subprocess.CalledProcessError as e: + status.reset('WARNING') + ODK_download_success = False #TODO: Check it came down cleanly ($serverOutputDir is created whether cleanly or not, so test more explicitly): ODK_csv_path = f"{jobPath}/ExportCSV/" - Path(ODK_csv_path).mkdir(parents=True, exist_ok=True) + Path(ODK_csv_path).mkdir(parents = True, exist_ok = True) ODK_csv_filename = f"SurveyData_{form_credentials['form_id']}.csv" - if ODK_download_success: + if ODK_download_success and not skip_download: + description_short = 'ODK export' description_long = 'converting ODK download to csv' logger.debug(description_long) @@ -100,9 +104,9 @@ def get_ODK_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat status.reset('WARNING') ODK_download_success = False - if not ODK_download_success: + if not ODK_download_success or skip_download: - logger.info("Because ODK server download failed somewhere, trying to recover by copying recent download") + logger.info("Because ODK server download failed somewhere (or we are skipping downloads), trying to recover by copying recent download") ODK_copy_success = False @@ -131,6 +135,7 @@ def get_ODK_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat past_ODK_csv_filename = ODK_csv_filename logger.info(f"Looking for {past_ODK_csv_path+past_ODK_csv_filename}") + print(f"Looking for {past_ODK_csv_path+past_ODK_csv_filename}") copyfile(past_ODK_csv_path+past_ODK_csv_filename,ODK_csv_path+ODK_csv_filename) @@ -775,7 +780,7 @@ def process_in_job_survey(jobPath,status,config,component): credentials_filename = config['Survey']['ServerCredentialsFile'] with open(credentials_filename) as credentials_file: - cred = json.load(credentials_file) + cred: dict = json.load(credentials_file) assert 'forms' in cred.keys() diff --git a/coordinator/ProcessorUtils.py b/coordinator/ProcessorUtils.py index 9dddef90302f7043d10931d850652e6da50f9375..e8ce959607a3c850a871a12844e902d22d24015a 100644 --- a/coordinator/ProcessorUtils.py +++ b/coordinator/ProcessorUtils.py @@ -10,6 +10,7 @@ from string import Template import subprocess import sys import tarfile +from typing import List short_name = { @@ -214,14 +215,14 @@ def query_proceed(necessary_file,description): return True -def query_past_successes(input_args): +def query_past_successes(input_args: dict): '''Checks if deposition and environment jobs are already completed successfully. If not, it raises an error.''' - component = input_args.component + component: str = input_args['component'] # check configs can be loaded - config_fns = input_args.config_paths + config_fns: List[str] = input_args['config_paths'] for configFile in config_fns: try: config_i = open_and_check_config(configFile) @@ -230,7 +231,7 @@ def query_past_successes(input_args): endScript(premature=True) # some config initialisation is necessary - config_i['StartString'] = input_args.start_date + config_i['StartString'] = input_args['start_date'] # check if deposition data is readily available dep_success_file = Template(config_i[component]['Deposition']['SuccessFileTemplate']).substitute(**config_i) diff --git a/tests/integration/test_survey.py b/tests/integration/test_survey.py new file mode 100644 index 0000000000000000000000000000000000000000..dd5d0426a47f829b5db169c792665a9f51b2e0d7 --- /dev/null +++ b/tests/integration/test_survey.py @@ -0,0 +1,47 @@ +import copy +import os +import unittest + +from integration.integration_test_utils import IntegrationTestUtils + + +class TestEnvSuit(unittest.TestCase): + + def setUp(self) -> None: + super().setUp() + default_config = '../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json' + self.default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config) + + def test_env_suit_standard_inputs_expected_results1(self): + + # nowstring: str = IntegrationTestUtils.get_now_string() + nowstring: str = "" + + os.environ["EMAIL_CRED"] = "../test_data/test_deployment/envs/Cred_gmail.json" + from Processor import run_Process, set_log_level + args_dict: dict = {} + args_dict['component'] = 'Survey' + args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_PATH] + args_dict['log_level'] = 'info' + args_dict['live'] = False + args_dict['start_date'] = '20221001' + args_dict['noupload'] = True + set_log_level(args_dict['log_level']) + + run_dict: dict = copy.deepcopy(self.default_config_dict) + test_out_path = run_dict['WorkspacePathout'] + nowstring + os.sep + run_dict['WorkspacePathout'] = test_out_path + run_dict['WorkspacePath'] = test_out_path + run_dict['Survey']['SkipServerDownload'] = True + run_dict['ServerName'] = '' # nothing, as local machine + + # run_dict['SubRegionNames'].remove('Kenya') + + IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_PATH) + + run_Process(args_dict) + self.assertTrue(True) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/SURVEYDATA_MANUAL/LIVE_SURVEYDATA_TOUSE.csv b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/SURVEYDATA_MANUAL/LIVE_SURVEYDATA_TOUSE.csv new file mode 100644 index 0000000000000000000000000000000000000000..d4e06f739c6b5d5343a841dfa6a6395395f87006 --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/SURVEYDATA_MANUAL/LIVE_SURVEYDATA_TOUSE.csv @@ -0,0 +1,2 @@ +SubmissionDate,start,end,today,deviceid,subscriberid,imei,phonenumber,username,surveyor_infromation-country,surveyor_infromation-surveyor_name,surveyor_infromation-institution,survey_infromation-location_name,survey_infromation-location-Latitude,survey_infromation-location-Longitude,survey_infromation-location-Altitude,survey_infromation-location-Accuracy,survey_infromation-survey_date,site_information-survey_site,site_information-crop,site_information-field_area,site_information-variety,site_information-growth_stage,other_crop,stem_rust-stemrust_incidence,stem_rust-Stemrust_severity,stem_rust-stemrust_host_plant_reaction,leaf_rust-leafrust_incidence,leaf_rust-leafrust_severity,leaf_rust-leafrust_host_plant_reaction,yellow_rust-yellowrust_incidence,yellow_rust-yellowrust_severity,yellow_rust-yellowrust_host_plant_reaction,septoria-septoria_incidence,septoria-septoria_severity,other_diseases_group-other_diseases,score_diseases_count,SET-OF-score_diseases,samples_collected,samples_type,sample_size-number_stemrust_live,sample_size-number_stemrust_dead_dna,sample_size-number_yellowrust_live,sample_size-number_yellowrust_dead,sample_size-number_leafrust_live,sample_size-using_barcode,live_stemrust_samples_count,SET-OF-live_stemrust_samples,dead_stemrust_samples_count,SET-OF-dead_stemrust_samples,live_yellowrust_samples_count,SET-OF-live_yellowrust_samples,dead_yellowrust_samples_count,SET-OF-dead_yellowrust_samples,live_leafrust_samples_count,SET-OF-live_leafrust_samples,comment,meta-instanceID,KEY +17-09-21 8:00,10-09-21 9:00,10-09-21 9:03,10-Sep-21,collect:1qNmuTci1M7xyjou,,collect:1qNmuTci1M7xyjou,,debrebirhan,Ethiopia,Kalkidan Yalew,Debre Birhan,Hagere Mariyam,9.2610515,39.4485883,2841.798221,3.95,10-Sep-21,farmer_field,bread_wheat,0.25,Kekeba,tillering,,none,9,na,none,9,na,none,9,na,low,11,,0,uuid:f3896fd5-741c-4ef7-8497-cce9f395a931/score_diseases,no,,,,,,,,,,,,,,,,,,A wheat field is found in less than 20m distance far from the inspected field,uuid:f3896fd5-741c-4ef7-8497-cce9f395a931,uuid:f3896fd5-741c-4ef7-8497-cce9f395a931 diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/SURVEYDATA_MANUAL/SurveyDataErrorsToRemove.csv b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/SURVEYDATA_MANUAL/SurveyDataErrorsToRemove.csv new file mode 100644 index 0000000000000000000000000000000000000000..41e8e7ffb35148c43c979d18c5511755530a8b48 --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/SURVEYDATA_MANUAL/SurveyDataErrorsToRemove.csv @@ -0,0 +1,2 @@ +SubmissionDate,start,end,today,deviceid,subscriberid,imei,phonenumber,username,surveyor_infromation-country,surveyor_infromation-surveyor_name,surveyor_infromation-institution,survey_infromation-location_name,survey_infromation-location-Latitude,survey_infromation-location-Longitude,survey_infromation-location-Altitude,survey_infromation-location-Accuracy,survey_infromation-survey_date,site_information-survey_site,site_information-crop,site_information-field_area,site_information-variety,site_information-growth_stage,other_crop,stem_rust-stemrust_incidence,stem_rust-Stemrust_severity,stem_rust-stemrust_host_plant_reaction,leaf_rust-leafrust_incidence,leaf_rust-leafrust_severity,leaf_rust-leafrust_host_plant_reaction,yellow_rust-yellowrust_incidence,yellow_rust-yellowrust_severity,yellow_rust-yellowrust_host_plant_reaction,septoria-septoria_incidence,septoria-septoria_severity,other_diseases_group-other_diseases,score_diseases_count,SET-OF-score_diseases,samples_collected,samples_type,sample_size-number_stemrust_live,sample_size-number_stemrust_dead_dna,sample_size-number_yellowrust_live,sample_size-number_yellowrust_dead,sample_size-number_leafrust_live,sample_size-using_barcode,live_stemrust_samples_count,SET-OF-live_stemrust_samples,dead_stemrust_samples_count,SET-OF-dead_stemrust_samples,live_yellowrust_samples_count,SET-OF-live_yellowrust_samples,dead_yellowrust_samples_count,SET-OF-dead_yellowrust_samples,live_leafrust_samples_count,SET-OF-live_leafrust_samples,comment,meta-instanceID,KEY +"Aug 6, 2018 12:43:06 PM","Jun 13, 2018 11:26:53 AM","Aug 6, 2018 12:42:21 PM","Jun 13, 2018",FC:19:10:89:E2:6B,"",FC:19:10:89:E2:6B,"",mekele,Ethiopia,tesfay gebrekirstos,Tigray Agricultural Research Institute,Ilala,13.5235519900,39.5033560800,1991.0000000000,5.0000000000,"Aug 6, 2018",farmer_field,bread_wheat,2.0000000000,mekele-1,tillering,"",high,20,mr,low,10,mr,high,50,s,medium,55,glume_blotch loose_smut pythium_root_rot scab_fusarium_head_blight take_all tan_spot,6,uuid:58cb6a0f-8e47-4339-8080-0e320234415c/score_diseases,no,"","","","","","","","","","","","","","","","","",There is potential inoculum for the future.,uuid:58cb6a0f-8e47-4339-8080-0e320234415c,uuid:58cb6a0f-8e47-4339-8080-0e320234415c diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/Cred-ODK-EIAR.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/Cred-ODK-EIAR.json new file mode 100644 index 0000000000000000000000000000000000000000..00d528703a6cc8068dda52618955ed9190fcf34b --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/Cred-ODK-EIAR.json @@ -0,0 +1,31 @@ +{ + "forms" : [ + { + "type" : "ODK", + "ODK_jar" : "./test_survey.py", + "url" : "not_needed", + "user" : "not_needed", + "pass" : "not_needed", + "form_id" : "wheat_rust_survey_1_0", + "form_name" : "Wheat rust survey 1_0" + }, + { + "type" : "kobotoolbox", + "url" : "not_needed", + "user" : "not_needed", + "pass" : "not_needed", + "form_token" : "not_needed", + "form_id" : "akpyJHvYxkLKPkxFJnPyTW", + "form_name" : "Wheat rust survey 1.0" + }, + { + "type" : "WRSIS", + "url" : "not_needed", + "user" : "not_needed", + "pass" : "not_needed", + "form_id" : "WRSIS", + "form_name" : "WRSIS" + } + + ] +} \ No newline at end of file diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json index 702d9bb420866b7d40c767639faec01608fa0cf5..72b7d0183ec997949dca1f248892a1a7591e485e 100644 --- a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json @@ -12,19 +12,24 @@ "Survey" : { "ProcessPreJob" : "process_pre_job_survey", "ProcessInJob" : "process_in_job_survey", - "ProcessEWSPlotting" : "process_EWS_plotting_survey", + "ProcessEWSPlotting": "process_EWS_plotting_survey", + "AcceptableDowntimeDays": 35, + "SeasonStartString" : "20220930", + "SkipServerDownload" : true, "ServerCredentialsFile" : "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/Cred-ODK-EIAR.json", "ServerPathExtra" : "/storage/sftp/metofficeupload/upload/Ethiopia/toMO/", "FormEdits" : { "wheat_rust_survey_1_0" : {}, "akpyJHvYxkLKPkxFJnPyTW" : { - "filter": { - "surveyor_infromation-country" : "Kenya" + "filter_by_list": { + "surveyor_infromation-country" : ["Kenya", "Ethiopia"] } - } + }, + "WRSIS" : {} }, "SourcesConfigFilename" : "../test_data/test_deployment/regions/EastAfrica/resources/wheat_source_generation/configs/config_EastAfrica_mapspam2017.R", - "SourcesRegionName" : "EastAfrica" + "SourcesRegionName" : "EastAfrica", + "pySourcesConfigFilename" : "../test_data/test_deployment/regions/EastAfrica/resources/source_gen/configs/config_EastAfrica_mapspam2017.json" }, "Environment" : { "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/Ethiopia/fromMO/daily_name/", @@ -164,14 +169,14 @@ "VariableNames" : ["P_GRAMINIS_DEPOSITION"], "PathTemplate" : "${WorkspacePath}DEPOSITION_${DateString}/WR_NAME_Ethiopia_${DateString}_fc/", "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS", - "FileListerFunction" : "list_deposition_files_historical", - "FileNameTemplate" : "deposition_srcs_allregions_C1_T${tIndexPlusOneString}_${iTimeString}.txt", + "FileListerFunction" : "list_files_split_historical_operational", + "FileNameTemplate" : "deposition_srcs_allregions_${DateString}.nc", "FileNamePrepared" : "?" }, "Environment" : { "PathTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${DateString}/processed/${RegionName}/${DiseaseName}/", "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS", - "FileListerFunction" : "list_env_suit_files_historical", + "FileListerFunction" : "list_files_split_historical_operational", "FileNameTemplate" : "RIE_value.csv", "FileNamePrepared" : "?" }, @@ -214,6 +219,7 @@ "RunConfig_seasonsofar" : "/../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/epi/RUN_CONFIG_EPI.json", "RunConfig_seasonplusforecast" : "../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/epi/RUN_CONFIG_EPI.json", "ChartConfig" : "../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_ETHIOPIA_PINE.json", + "PlottingRegionName" : "Ethiopia", "EpiCase" : "ps" } }, @@ -234,15 +240,22 @@ }, "Aggregation" : { "raster_path_template" : "${envDataPath}/suitability_${SubRegionNameLower}_${disease_name_snakecase}_total.csv", - "raster_name_template" : "RIE_${disease_name_camelcase}", - "shape_fn_entry" : "ShapeFilenameAdmin1", - "shape_coln" : "ZONENAME", + "raster_name_template" : "RIE_${disease_name_camelcase}_%", + "shape_fn_entry" : "ShapeFilenameAdmin2", + "shape_coln" : "ADM2_EN", + "show_columns" : ["ADM0_EN","ADM1_EN"], + "separate_rows_for_coln":"ADM0_EN", "func" : "mean", - "save_path_template" : "${jobPath}/images/suitability_${SubRegionNameLower}_${disease_name_snakecase}_mean_per_zone" + "multiply_by" : 100, + "float_format" : ".0f", + "display_format" : ".0f", + "save_path_template" : "${jobPath}/images/suitability_${SubRegionNameLower}_${disease_name_snakecase}_mean_per_zone", + "max_entries" : 20 } }, "Deposition" : { "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS", + "DataPathTemplate" : "${WorkspacePath}/DEPOSITION_${dateString}/plotting/${SubRegionNameLower}/input_csvs/", "PlotPathTemplate" : "${WorkspacePath}/DEPOSITION_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/", "PlotFilenameTemplates" : { "stripe": "${depPlotPath}/deposition_${SubRegionNameLower}_stripe_rust_total_${dateString}0000_${weekAheadString}0000_map.png", @@ -250,14 +263,18 @@ "leaf": "${depPlotPath}/deposition_${SubRegionNameLower}_leaf_rust_total_${dateString}0000_${weekAheadString}0000_map.png" }, "Aggregation" : { - "raster_path_template" : "${depPlotPath}/../../input_csvs/deposition_${SubRegionNameLowerShort}_${disease_name_snakecase}_total.csv", + "raster_path_template" : "${depDataPath}/deposition_${SubRegionNameLower}_${disease_name_snakecase}_total.csv", "raster_name_template" : "spores_per_m2_per_week_${disease_name_camelcase}", - "shape_fn_entry" : "ShapeFilenameAdmin1", - "shape_coln" : "ADM1_EN", - "show_columns" : ["ADM0_EN","REGIONNAME"], + "shape_fn_entry" : "ShapeFilenameAdmin2", + "shape_coln" : "ADM2_EN", + "show_columns" : ["ADM0_EN","ADM1_EN"], "separate_rows_for_coln":"ADM0_EN", "func" : "mean", - "save_path_template" : "${jobPath}/images/deposition_${SubRegionNameLower}_${disease_name_snakecase}_mean_per_zone" + "multiply_by" : 1, + "float_format" : ".1e", + "display_format" : ".0f", + "save_path_template" : "${jobPath}/images/deposition_${SubRegionNameLower}_${disease_name_snakecase}_mean_per_zone", + "max_entries" : 20 } }, "Surveys" : { diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/approx_growth_timing_cleaned.csv b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/approx_growth_timing_cleaned.csv new file mode 100644 index 0000000000000000000000000000000000000000..5a6c122c6a06b67559f1123ab565255fca248e94 --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/approx_growth_timing_cleaned.csv @@ -0,0 +1,11 @@ +Remaining_days_Spring,Remaining_days_Winter,GrowthStageName +110,130,tillering +60,90,boot +50,50,heading +40,40,flowering +30,40,milk +20,40,dough +7,7,maturity +40,40,NA +40,40,na +40,40,n/a diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.cpg b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.cpg new file mode 100644 index 0000000000000000000000000000000000000000..3ad133c048f2189041151425a73485649e6c32c0 --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.cpg @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.dbf b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.dbf new file mode 100644 index 0000000000000000000000000000000000000000..0fa734a5cf756be7097af5ae67a91bcc45b0e9f0 Binary files /dev/null and b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.dbf differ diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.prj b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.prj new file mode 100644 index 0000000000000000000000000000000000000000..f45cbadf0074d8b7b2669559a93bc50bb95f82d4 --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.prj @@ -0,0 +1 @@ +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]] \ No newline at end of file diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.qmd b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.qmd new file mode 100644 index 0000000000000000000000000000000000000000..ba33e81388ee55ee924721615a1646dc65fdde37 --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.qmd @@ -0,0 +1,26 @@ +<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'> +<qgis version="3.22.9-Bia?owie?a"> + <identifier></identifier> + <parentidentifier></parentidentifier> + <language></language> + <type>dataset</type> + <title></title> + <abstract></abstract> + <links/> + <fees></fees> + <encoding></encoding> + <crs> + <spatialrefsys> + <wkt></wkt> + <proj4></proj4> + <srsid>0</srsid> + <srid>0</srid> + <authid></authid> + <description></description> + <projectionacronym></projectionacronym> + <ellipsoidacronym></ellipsoidacronym> + <geographicflag>false</geographicflag> + </spatialrefsys> + </crs> + <extent/> +</qgis> diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.shp b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.shp new file mode 100644 index 0000000000000000000000000000000000000000..0838539aafd0c41fb9b7abb4dd926b4adfd451bc Binary files /dev/null and b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.shp differ diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.shx b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.shx new file mode 100644 index 0000000000000000000000000000000000000000..86e78eef05852d982321499463ae73cdf52f8f86 Binary files /dev/null and b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters_small.shx differ diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.cpg b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.cpg new file mode 100644 index 0000000000000000000000000000000000000000..3ad133c048f2189041151425a73485649e6c32c0 --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.cpg @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.dbf b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.dbf new file mode 100644 index 0000000000000000000000000000000000000000..1ba6125a95a1578a934ce7727af07a19799034c7 Binary files /dev/null and b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.dbf differ diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.prj b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.prj new file mode 100644 index 0000000000000000000000000000000000000000..f45cbadf0074d8b7b2669559a93bc50bb95f82d4 --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.prj @@ -0,0 +1 @@ +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]] \ No newline at end of file diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.qmd b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.qmd new file mode 100644 index 0000000000000000000000000000000000000000..bedd3f27965d154f6d092c5a5641c7161ea5cd9e --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.qmd @@ -0,0 +1,26 @@ +<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'> +<qgis version="3.22.9-Bia?owie?a"> + <identifier></identifier> + <parentidentifier></parentidentifier> + <language></language> + <type></type> + <title></title> + <abstract></abstract> + <links/> + <fees></fees> + <encoding></encoding> + <crs> + <spatialrefsys> + <wkt></wkt> + <proj4></proj4> + <srsid>0</srsid> + <srid>0</srid> + <authid></authid> + <description></description> + <projectionacronym></projectionacronym> + <ellipsoidacronym></ellipsoidacronym> + <geographicflag>false</geographicflag> + </spatialrefsys> + </crs> + <extent/> +</qgis> diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.shp b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.shp new file mode 100644 index 0000000000000000000000000000000000000000..dd396266a0c8df1c2d8e020239f5ec1b14c1cb5d Binary files /dev/null and b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.shp differ diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.shx b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.shx new file mode 100644 index 0000000000000000000000000000000000000000..da1ac25b5c87a3e47a677833e6d70658af57e31f Binary files /dev/null and b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_regions_small.shx differ diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.cpg b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.cpg new file mode 100644 index 0000000000000000000000000000000000000000..cd89cb9758e84c59b13399a25325d366af87e4f8 --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.cpg @@ -0,0 +1 @@ +ISO-8859-1 \ No newline at end of file diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.csv b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.csv new file mode 100644 index 0000000000000000000000000000000000000000..df02ddc5e9cc1846cab50ad93bd2cb9dd9552b18 --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.csv @@ -0,0 +1,4 @@ +,id,REGION_ID,wheat_ha +0,1,1,32090 +1,2,2,18463 +2,3,1,27731 diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.dbf b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.dbf new file mode 100644 index 0000000000000000000000000000000000000000..361003b9f4acdc3122802fe67eb839b01166374f Binary files /dev/null and b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.dbf differ diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.prj b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.prj new file mode 100644 index 0000000000000000000000000000000000000000..8042941d32058bf9cf23290a7383498a66449581 --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.prj @@ -0,0 +1 @@ +PROJCS["WGS_1984_World_Mercator",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],UNIT["Meter",1.0]] \ No newline at end of file diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.shp b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.shp new file mode 100644 index 0000000000000000000000000000000000000000..3973b971cb278316bca77f88d0f3c37ae7759e51 Binary files /dev/null and b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.shp differ diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.shx b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.shx new file mode 100644 index 0000000000000000000000000000000000000000..f25cbb40952d8a1b7121d3a2991f02ae40ba23ac Binary files /dev/null and b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.shx differ diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/spam2017V1r1_SSA_gr_H_WHEA_A_rescaled_oneliner.tif b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/spam2017V1r1_SSA_gr_H_WHEA_A_rescaled_oneliner.tif new file mode 100644 index 0000000000000000000000000000000000000000..dd407a5b68180e5cb483f21d882cd795e93358e9 Binary files /dev/null and b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/preprocessed/spam2017V1r1_SSA_gr_H_WHEA_A_rescaled_oneliner.tif differ diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/spam2017V1r1_SSA_gr_H_WHEA_A_small.tif b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/spam2017V1r1_SSA_gr_H_WHEA_A_small.tif new file mode 100644 index 0000000000000000000000000000000000000000..13beef725083378f9e8c637f50931a20bb788e88 Binary files /dev/null and b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/assets/spam2017V1r1_SSA_gr_H_WHEA_A_small.tif differ diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/configs/config_EastAfrica_mapspam2017.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/configs/config_EastAfrica_mapspam2017.json new file mode 100644 index 0000000000000000000000000000000000000000..b5106eb9a5b6b63457b177ac9926345bcf12ac3e --- /dev/null +++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/source_gen/configs/config_EastAfrica_mapspam2017.json @@ -0,0 +1,14 @@ +{ + "config_name" : "EastAfrica", + "region_file" : "/media/scratch/lb584_scratch/projects/ews_local_prod/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_9regions.shp", + "wheat_source_name" : "MAPSPAM2017", + "wheat_file" : "/media/scratch/lb584_scratch/projects/ews_local_prod/regions/EastAfrica/resources/source_gen/assets/spam2017V1r1_SSA_gr_H_WHEA_A.tif", + "wheat_file_rescaled" : "/media/scratch/lb584_scratch/projects/ews_local_prod/regions/EastAfrica/resources/source_gen/assets/preprocessed/spam2017V1r1_SSA_gr_H_WHEA_A_rescaled_oneliner.tif", + "cluster_poly_file_no_wheat" : "/media/scratch/lb584_scratch/projects/ews_local_prod/regions/EastAfrica/resources/source_gen/assets/boundaries/EastAfrica_clusters.shp", + "cluster_poly_file" : "//media/scratch/lb584_scratch/projects/ews_local_prod/regions/EastAfrica/resources/source_gen/assets/preprocessed/EastAfrica_clusters_with_wheat_ha_MAPSPAM2017.shp", + "surveyor_name_col" : "surveyor_infromation-surveyor_name", + "id_col" : "id", + "region_id_col" : "REGION_ID", + "wheat_stages_file" : "/media/scratch/lb584_scratch/projects/ews_local_prod/regions/EastAfrica/resources/source_gen/assets/approx_growth_timing_cleaned.csv" +} +