Merge remote-tracking branch 'origin/main' into main

coordinator/ProcessorSurveys.py

@@ -370,7 +370,7 @@ class ProcessorSurveys(Processor):
             origins_list = df_join["Origin"].unique()
             groups = {i:[i] for i in origins_list}
 
-            assert not np_any([k in origins_list for k in config['Survey']['Groups'].keys()])
+            assert not np_any([k in origins_list for k in config['Survey']['Groups'].keys()]), 'Group(s) defined in config already present in the survey data as origin'
 
             groups.update(config['Survey']['Groups'])
 

coordinator/ProcessorSurveysWRT.py

@@ -20,6 +20,27 @@ from ProcessorUtils import (
 logger = logging.getLogger('Processor.Surveys.WRT')
 add_filters_to_sublogger(logger)
 
+# Downloads of the WRT based on the ODK form reports country descriptions as two letter codes.
+# The reason is unclear, as viewing online shows full country names. To be
+# consistent across all form types, this function is available for translating
+# two letter codes to full string names.
+# Only some have these have been checked explicitly against downloaded surveys.
+# Assuming the code is based on ISO 3166-1 alpha-2
+COUNTRY_ABBREVIATIONS_DICT = {
+        'AF' : 'Afghanistan',
+        'BD' : 'Bangladesh',
+        'BT' : 'Bhutan',
+        'ET' : 'Ethiopia',
+        'IN' : 'India',
+        'KE' : 'Kenya',
+        'NP' : 'Nepal',
+        'PK' : 'Pakistan',
+        'TZ' : 'Tanzania',
+        'UG' : 'Uganda',
+        'ZM' : 'Zambia',
+        'ZW' : 'Zimbabwe',
+        }
+
 # Cases to convert WRT categories to expected short name category or value
 cases_incident = {
     '':'none',
@@ -40,12 +61,12 @@ cases_severity = {
 def get_from_WRT(form_credentials: dict, startDate: str, endDate: str):
     '''Make the HTTP request, then checking the status code.'''
     
-    url = f"{form_credentials['url']}WheatRustAPI/api/Survey/GetAll"
-    #url = f"{form_credentials['url']}WheatRustAPI/api/Survey/GetByPublishedLevel"
+    #url = f"{form_credentials['url']}WheatRustAPI/api/Survey/GetAll"
+    url = f"{form_credentials['url']}WheatRustAPI/api/Survey/GetByPublishedLevel"
     url += '?strStopDate=' + endDate
     url += '&strStartDate=' + startDate
     url += '&strArrayCountryID='+ form_credentials['countryID']
-    #url += '&strArrayPublishedLevel=Unpubl'
+    url += '&strArrayPublishedLevel=' + form_credentials['publishedLevel']
     
     # set up http session
     session = requests.Session()
@@ -177,7 +198,7 @@ def get_WRT_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat
         "ObservationID" : 'KEY',
         "OriginalID" : 'None',
         "Origin" : "Origin",
-        "Country" : 'surveyor_infromation-country',
+        "Country" : ('parse_cases',(('name_out','surveyor_infromation-country'),('cases', COUNTRY_ABBREVIATIONS_DICT))),
         "Latitude" : 'survey_infromation-location-Latitude',
         "Longitude" : 'survey_infromation-location-Longitude',
         "FieldArea" : 'site_information-field_area',
@@ -226,9 +247,9 @@ def get_WRT_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat
             download_success = False
 
     # define filenames
-    csv_filename = f"SurveyData_raw.csv"
+    csv_filename = f"SurveyData_raw_{form_credentials['form_name']}.csv"
 
-    csv_processed_filename = f"SurveyDataProcessed.csv"
+    csv_processed_filename = f"SurveyDataProcessed_{form_credentials['form_name']}.csv"
     csv_processed_path = f"{output_path}/{csv_processed_filename}"
 
     if download_success and not skip_download:

tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json

@@ -236,6 +236,11 @@
         "ProcessInJob" : "process_in_job_advisory",
         "ProcessEWSPlotting" : "do_nothing",
         "seasonStartString" : "20210901",
+        "SurveyPathTemplate" : "${WorkspacePath}/SURVEYDATA_${dateString}/ExportCSV/Processed_SurveyData.csv",
+        "DependentComponents" : [
+            "Deposition",
+            "Environment"
+         ],
         "Environment" : {
             "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS",
             "DataPathTemplate" : "${WorkspacePath}/ENVIRONMENT_2.0_${dateString}/plotting/${SubRegionNameLower}/input_nc/",
@@ -284,15 +289,15 @@
                 "max_entries" : 20
             }
         },
-        "Epidemiology" : {
+        "NOT_USING_Epidemiology_NOT_USING" : {
             "SuccessFileTemplate" : "${WorkspacePath}EPI_${StartString}/STATUS_SUCCESS",
             "DataPathTemplate" : "${WorkspacePath}/EPI_${dateString}/plotting/input_csvs/",
             "PlotPathTemplate" : "${WorkspacePath}/EPI_${dateString}/plotting/images/",
             "PlotFilenameTemplates" : {
-                "stem": "${depPlotPath}/infection_${SubRegionNameLower}_stem_seasonincforecast_*0300_*0000_map.png"
+                "stem": "${epiPlotPath}/infection_${SubRegionNameLower}_stem_seasonincforecast_*0300_*0000_map.png"
             },
             "Aggregation" : {
-                "raster_path_template" : "${depDataPath}/${SubRegionNameLower}_${disease_name_snakecase}_masked.nc",
+                "raster_path_template" : "${epiDataPath}/${SubRegionNameLower}_${disease_name_snakecase}_masked.nc",
                 "raster_name_template" : "predicted_infected_area_${disease_name_camelcase}",
                 "shape_fn_entry" : "ShapeFilenameAdmin2",
                 "shape_coln" : "ADM2_EN",
@@ -310,14 +315,18 @@
             "variety_names_tidy_fn" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/configs/config_EastAfrica_variety_names_tidy.json",
             "CountryColumnName" : "surveyor_infromation-country",
             "EastAfrica" : {
-                "ShapeFilenameAdmin0" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
+                "CountryList" : ["Ethiopia", "Kenya", "Tanzania", "Zambia", "Uganda"],
                 "ShapeFilenameAdmin1" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
-                "ShapeFilenameAdmin2" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp"
+                "ShapeFilenameAdmin2" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
+                "ShapeFilenameToAggregate" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
+                "Extent" : [21.5, 47, -18.5, 15],
+                "LabelZoom" : 5
             },
             "Ethiopia" : {
-                "ShapeFilenameAdmin0" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
                 "ShapeFilenameAdmin1" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
-                "ShapeFilenameAdmin2" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp"
+                "ShapeFilenameAdmin2" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
+                "Extent" : [32.8, 47, 3.2, 15],
+                "LabelZoom" : 7
             }
         },
         "logos" : "../../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/logos_Ethiopia_plotting.png"