fix: handle nans more robust

coordinator/ProcessorSurveysnewODK.py

@@ -22,7 +22,6 @@ add_filters_to_sublogger(logger)
 
 # Cases to convert incident categories to expected short name category or value
 cases_incident = {
-    '':'none',
     'N':'none',
     'L':'low',
     'M':'medium',
@@ -34,7 +33,7 @@ cases_incident = {
     }
 
 cases_severity = {
-    '' : 0,
+    '0':0,
     '1':1,
     '5':5,
     '10':10,
@@ -45,7 +44,7 @@ cases_severity = {
     '50':50,
     '60':60,
     '70':70,
-    '80':80,
+    '80':80,   
     '80+':80,
     }
 
@@ -124,8 +123,8 @@ def get_newODK_form_as_csv(form_credentials: dict, jobPath: str, config: dict, s
         'formhub/uuid' : 'None',
         'imei' : 'imei',
         'leaf_rust/leafrust_host_plant_reaction' : 'leaf_rust-leafrust_host_plant_reaction',
-        'leaf_rust/leafrust_incidence' : ('parse_cases',(('name_out','leaf_rust-leafrust_incidence'),('cases', cases_incident))),
-        'leaf_rust/leafrust_severity' : ('parse_cases',(('name_out','leaf_rust-leafrust_severity'),('cases', cases_severity),('dtype', int))),
+        'leaf_rust/leafrust_incidence' : ('parse_cases',(('name_out','leaf_rust-leafrust_incidence'),('cases', cases_incident),('fillna','none'))),
+        'leaf_rust/leafrust_severity' : ('parse_cases',(('name_out','leaf_rust-leafrust_severity'),('cases', cases_severity),('dtype', int),('fillna','0'))),
         'live_leafrust_samples' : 'SET-OF-live_leafrust_samples',
         'live_leafrust_samples_count' : 'live_leafrust_samples_count',
         'live_stemrust_samples' : 'SET-OF-live_stemrust_samples',
@@ -143,9 +142,9 @@ def get_newODK_form_as_csv(form_credentials: dict, jobPath: str, config: dict, s
         'site_information/survey_site' : 'site_information-survey_site',
         'site_information/variety' : 'site_information-variety',
         'start' : ('parse_date',(('name_out','start'),('fmt_in','%Y-%m-%dT%H:%M:%S.%f%z'))),
-        'stem_rust/Stemrust_severity' : ('parse_cases',(('name_out','stem_rust-Stemrust_severity'),('cases', cases_severity),('dtype', int))),
+        'stem_rust/Stemrust_severity' : ('parse_cases',(('name_out','stem_rust-Stemrust_severity'),('cases', cases_severity),('dtype', int),('fillna','0'))),
         'stem_rust/stemrust_host_plant_reaction' : 'stem_rust-stemrust_host_plant_reaction',
-        'stem_rust/stemrust_incidence' : ('parse_cases',(('name_out','stem_rust-stemrust_incidence'),('cases', cases_incident))),
+        'stem_rust/stemrust_incidence' : ('parse_cases',(('name_out','stem_rust-stemrust_incidence'),('cases', cases_incident),('fillna','none'))),
         'survey_infromation/location' : ('parse_location_kobotoolbox',()),
         'survey_infromation/location_name' : 'survey_infromation-location_name',
         'survey_infromation/survey_date' : ('parse_date',(('name_out','survey_infromation-survey_date'),('fmt_in','%Y-%m-%d'))),
@@ -155,8 +154,8 @@ def get_newODK_form_as_csv(form_credentials: dict, jobPath: str, config: dict, s
         'today' : ('parse_date',(('name_out','today'),('fmt_in','%Y-%m-%d'))),
         'username' : 'username',
         'yellow_rust/yellowrust_host_plant_reaction' : 'yellow_rust-yellowrust_host_plant_reaction',
-        'yellow_rust/yellowrust_incidence' : ('parse_cases',(('name_out','yellow_rust-yellowrust_incidence'),('cases', cases_incident))),
-        'yellow_rust/yellowrust_severity' : ('parse_cases',(('name_out','yellow_rust-yellowrust_severity'),('cases', cases_severity),('dtype', int))),
+        'yellow_rust/yellowrust_incidence' : ('parse_cases',(('name_out','yellow_rust-yellowrust_incidence'),('cases', cases_incident),('fillna','none'))),
+        'yellow_rust/yellowrust_severity' : ('parse_cases',(('name_out','yellow_rust-yellowrust_severity'),('cases', cases_severity),('dtype', int),('fillna','0'))),
         }
 
     logger.debug('Performing download')