Clean WRT processing

ews/coordinator/survey_servers/processor_surveys_wrt.py

@@ -112,51 +112,6 @@ def nested_to_flattened(df):
         for i in RAW_COLUMNS:
             df[i] = ""
 
-    # add new columns
-    logger.info('Adding new columns')
-    NEW_COLUMNS = ['imei',
-            'sample_size-number_yellowrust_live',
-            'sample_size-number_stemrust_live',
-            'dead_stemrust_samples_count',
-            'samples_collected',
-            'sample_size-number_yellowrust_dead',
-            'live_leafrust_samples_count',
-            'other_crop',
-            'live_yellowrust_samples_count',
-            'subscriberid',
-            'sample_size-using_barcode',
-            'start',
-            'score_diseases_count',
-            'phonenumber',
-            'survey_infromation-location-Accuracy',
-            'SET-OF-live_yellowrust_samples',
-            'SET-OF-score_diseases',
-            'meta-instanceID',
-            'deviceid',
-            'end',
-            'samples_type',
-            'live_stemrust_samples_count',
-            'dead_yellowrust_samples_count',
-            'SET-OF-live_leafrust_samples',
-            'other_diseases_group-other_diseases',
-            'survey_infromation-location-Altitude',
-            'SET-OF-dead_stemrust_samples',
-            'comment',
-            'sample_size-number_leafrust_live',
-            'today',
-            'SET-OF-dead_yellowrust_samples',
-            'username',
-            'SET-OF-live_stemrust_samples',
-            'sample_size-number_stemrust_dead_dna',
-            'SubmissionDate',
-            'survey_infromation-location_name',
-            'site_information-crop',
-            'surveyor_infromation-institution',
-            'surveyor_infromation-surveyor_name']
-
-    for i in NEW_COLUMNS:
-        df[i] = ""
-
     # add dedicated rust columns, with default values
     NEW_RUST_COLUMNS = {'SR.Incident':'N','SR.Severity':'N','SR.Reaction':'N', 'SR.Race':'', 'SR.Genotype':'',
                         'LR.Incident':'N','LR.Severity':'N','LR.Reaction':'N', 'LR.Race':'', 'LR.Genotype':'',
@@ -205,35 +160,35 @@ def get_WRT_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat
     column_parser_dict = {
         "ObservationID" : 'KEY',
         "OriginalID" : 'None',
-        "PublishedLevel" : 'PublishedLevel',
-        "Origin" : 'Origin',
-        "Country" : ('parse_cases',(('name_out','surveyor_infromation-country'),('cases', COUNTRY_ABBREVIATIONS_DICT))),
-        "Latitude" : 'survey_infromation-location-Latitude',
-        "Longitude" : 'survey_infromation-location-Longitude',
-        "FieldAreaHa" : ('parse_dtype',(('name_out','site_information-field_area'),('dtype', float))),
-        "Cultivar" : 'site_information-variety',
-        "CollectionDate" : ('parse_date',(('name_out','survey_infromation-survey_date'),('fmt_in','%m/%d/%Y'))),
+        "PublishedLevel" : 'published_level',
+        "Origin" : 'origin',
+        "Country" : ('parse_cases',(('name_out','country'),('cases', COUNTRY_ABBREVIATIONS_DICT))),
+        "Latitude" : 'latitude',
+        "Longitude" : 'longitude',
+        "FieldAreaHa" : ('parse_dtype',(('name_out','field_area'),('dtype', float))),
+        "Cultivar" : 'variety',
+        "CollectionDate" : ('parse_date',(('name_out','survey_date'),('fmt_in','%m/%d/%Y'))),
         "UpdateDate" : 'None',
-        "GrowthStage" : 'site_information-growth_stage',
-        "SurveySite" : 'site_information-survey_site',
+        "GrowthStage" : 'growth_stage',
+        "SurveySite" : 'survey_site',
         "listDisease" : 'None',
-        "YR.Severity" : ('parse_cases',(('name_out','yellow_rust-yellowrust_severity'),('cases', cases_severity),('dtype', int))),
-        "YR.Incident" : ('parse_cases',(('name_out','yellow_rust-yellowrust_incidence'),('cases', cases_incident))),
-        "YR.Reaction" : 'yellow_rust-yellowrust_host_plant_reaction',
+        "YR.Severity" : ('parse_cases',(('name_out','severity_yellow_rust'),('cases', cases_severity),('dtype', int))),
+        "YR.Incident" : ('parse_cases',(('name_out','incidence_yellow_rust'),('cases', cases_incident))),
+        "YR.Reaction" : 'None',
         "YR.Race" : 'None',
         "YR.Genotype" : 'None',
-        "SR.Severity" : ('parse_cases',(('name_out','stem_rust-Stemrust_severity'),('cases', cases_severity),('dtype', int))),
-        "SR.Incident" : ('parse_cases',(('name_out','stem_rust-stemrust_incidence'),('cases', cases_incident))),
-        "SR.Reaction" : 'stem_rust-stemrust_host_plant_reaction',
+        "SR.Severity" : ('parse_cases',(('name_out','severity_stem_rust'),('cases', cases_severity),('dtype', int))),
+        "SR.Incident" : ('parse_cases',(('name_out','incidence_stem_rust'),('cases', cases_incident))),
+        "SR.Reaction" : 'None',
         "SR.Race" : 'None',
         "SR.Genotype" : 'None',
-        "LR.Severity" : ('parse_cases',(('name_out','leaf_rust-leafrust_severity'),('cases', cases_severity),('dtype', int))),
-        "LR.Incident" : ('parse_cases',(('name_out','leaf_rust-leafrust_incidence'),('cases', cases_incident))),
-        "LR.Reaction" : 'leaf_rust-leafrust_host_plant_reaction',
+        "LR.Severity" : ('parse_cases',(('name_out','severity_leaf_rust'),('cases', cases_severity),('dtype', int))),
+        "LR.Incident" : ('parse_cases',(('name_out','incidence_leaf_rust'),('cases', cases_incident))),
+        "LR.Reaction" : 'None',
         "LR.Race" : 'None',
         "LR.Genotype" : 'None',
-        "Septoria.Severity" : 'septoria-septoria_severity',
-        "Septoria.Incident" : 'septoria-septoria_incidence'
+        "Septoria.Severity" : 'None',
+        "Septoria.Incident" : 'None'
     }
 
     # perform a pull from the server, and if it fails write a warning message