diff --git a/coordinator/ProcessorSurveys.py b/coordinator/ProcessorSurveys.py
index 33ab937ad26b3f7b64af53871f83652ef9e20c96..8da4d69c1eb536e9351337c747a67a29e3d70bb2 100644
--- a/coordinator/ProcessorSurveys.py
+++ b/coordinator/ProcessorSurveys.py
@@ -370,7 +370,7 @@ class ProcessorSurveys(Processor):
origins_list = df_join["Origin"].unique()
groups = {i:[i] for i in origins_list}
- assert not np_any([k in origins_list for k in config['Survey']['Groups'].keys()])
+ assert not np_any([k in origins_list for k in config['Survey']['Groups'].keys()]), 'Group(s) defined in config already present in the survey data as origin'
groups.update(config['Survey']['Groups'])
diff --git a/coordinator/ProcessorSurveysWRT.py b/coordinator/ProcessorSurveysWRT.py
index 12018382db477443b658351a9bfe1a5c780fea31..54446e4b281b71550d5a92e0bd5afe74930b6273 100644
--- a/coordinator/ProcessorSurveysWRT.py
+++ b/coordinator/ProcessorSurveysWRT.py
@@ -20,6 +20,27 @@ from ProcessorUtils import (
logger = logging.getLogger('Processor.Surveys.WRT')
add_filters_to_sublogger(logger)
+# Downloads of the WRT based on the ODK form reports country descriptions as two letter codes.
+# The reason is unclear, as viewing online shows full country names. To be
+# consistent across all form types, this function is available for translating
+# two letter codes to full string names.
+# Only some have these have been checked explicitly against downloaded surveys.
+# Assuming the code is based on ISO 3166-1 alpha-2
+COUNTRY_ABBREVIATIONS_DICT = {
+ 'AF' : 'Afghanistan',
+ 'BD' : 'Bangladesh',
+ 'BT' : 'Bhutan',
+ 'ET' : 'Ethiopia',
+ 'IN' : 'India',
+ 'KE' : 'Kenya',
+ 'NP' : 'Nepal',
+ 'PK' : 'Pakistan',
+ 'TZ' : 'Tanzania',
+ 'UG' : 'Uganda',
+ 'ZM' : 'Zambia',
+ 'ZW' : 'Zimbabwe',
+ }
+
# Cases to convert WRT categories to expected short name category or value
cases_incident = {
'':'none',
@@ -40,12 +61,12 @@ cases_severity = {
def get_from_WRT(form_credentials: dict, startDate: str, endDate: str):
'''Make the HTTP request, then checking the status code.'''
- url = f"{form_credentials['url']}WheatRustAPI/api/Survey/GetAll"
- #url = f"{form_credentials['url']}WheatRustAPI/api/Survey/GetByPublishedLevel"
+ #url = f"{form_credentials['url']}WheatRustAPI/api/Survey/GetAll"
+ url = f"{form_credentials['url']}WheatRustAPI/api/Survey/GetByPublishedLevel"
url += '?strStopDate=' + endDate
url += '&strStartDate=' + startDate
url += '&strArrayCountryID='+ form_credentials['countryID']
- #url += '&strArrayPublishedLevel=Unpubl'
+ url += '&strArrayPublishedLevel=' + form_credentials['publishedLevel']
# set up http session
session = requests.Session()
@@ -177,7 +198,7 @@ def get_WRT_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat
"ObservationID" : 'KEY',
"OriginalID" : 'None',
"Origin" : "Origin",
- "Country" : 'surveyor_infromation-country',
+ "Country" : ('parse_cases',(('name_out','surveyor_infromation-country'),('cases', COUNTRY_ABBREVIATIONS_DICT))),
"Latitude" : 'survey_infromation-location-Latitude',
"Longitude" : 'survey_infromation-location-Longitude',
"FieldArea" : 'site_information-field_area',
@@ -226,9 +247,9 @@ def get_WRT_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat
download_success = False
# define filenames
- csv_filename = f"SurveyData_raw.csv"
+ csv_filename = f"SurveyData_raw_{form_credentials['form_name']}.csv"
- csv_processed_filename = f"SurveyDataProcessed.csv"
+ csv_processed_filename = f"SurveyDataProcessed_{form_credentials['form_name']}.csv"
csv_processed_path = f"{output_path}/{csv_processed_filename}"
if download_success and not skip_download:
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
index 3967c5d67db3415ba799965e0e9fe0edfbdad0bb..aecc5f8ba3b5b53e69d223618fc0dcfa7886afa0 100644
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
@@ -236,6 +236,11 @@
"ProcessInJob" : "process_in_job_advisory",
"ProcessEWSPlotting" : "do_nothing",
"seasonStartString" : "20210901",
+ "SurveyPathTemplate" : "${WorkspacePath}/SURVEYDATA_${dateString}/ExportCSV/Processed_SurveyData.csv",
+ "DependentComponents" : [
+ "Deposition",
+ "Environment"
+ ],
"Environment" : {
"SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS",
"DataPathTemplate" : "${WorkspacePath}/ENVIRONMENT_2.0_${dateString}/plotting/${SubRegionNameLower}/input_nc/",
@@ -284,15 +289,15 @@
"max_entries" : 20
}
},
- "Epidemiology" : {
+ "NOT_USING_Epidemiology_NOT_USING" : {
"SuccessFileTemplate" : "${WorkspacePath}EPI_${StartString}/STATUS_SUCCESS",
"DataPathTemplate" : "${WorkspacePath}/EPI_${dateString}/plotting/input_csvs/",
"PlotPathTemplate" : "${WorkspacePath}/EPI_${dateString}/plotting/images/",
"PlotFilenameTemplates" : {
- "stem": "${depPlotPath}/infection_${SubRegionNameLower}_stem_seasonincforecast_*0300_*0000_map.png"
+ "stem": "${epiPlotPath}/infection_${SubRegionNameLower}_stem_seasonincforecast_*0300_*0000_map.png"
},
"Aggregation" : {
- "raster_path_template" : "${depDataPath}/${SubRegionNameLower}_${disease_name_snakecase}_masked.nc",
+ "raster_path_template" : "${epiDataPath}/${SubRegionNameLower}_${disease_name_snakecase}_masked.nc",
"raster_name_template" : "predicted_infected_area_${disease_name_camelcase}",
"shape_fn_entry" : "ShapeFilenameAdmin2",
"shape_coln" : "ADM2_EN",
@@ -310,14 +315,18 @@
"variety_names_tidy_fn" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/configs/config_EastAfrica_variety_names_tidy.json",
"CountryColumnName" : "surveyor_infromation-country",
"EastAfrica" : {
- "ShapeFilenameAdmin0" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
+ "CountryList" : ["Ethiopia", "Kenya", "Tanzania", "Zambia", "Uganda"],
"ShapeFilenameAdmin1" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
- "ShapeFilenameAdmin2" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp"
+ "ShapeFilenameAdmin2" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
+ "ShapeFilenameToAggregate" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
+ "Extent" : [21.5, 47, -18.5, 15],
+ "LabelZoom" : 5
},
"Ethiopia" : {
- "ShapeFilenameAdmin0" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
"ShapeFilenameAdmin1" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
- "ShapeFilenameAdmin2" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp"
+ "ShapeFilenameAdmin2" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
+ "Extent" : [32.8, 47, 3.2, 15],
+ "LabelZoom" : 7
}
},
"logos" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/logos_Ethiopia_plotting.png"