diff --git a/ews/coordinator/survey_servers/processor_surveys_wrt.py b/ews/coordinator/survey_servers/processor_surveys_wrt.py index 813056934f2df8d812f55915b31e22b00d799a22..3bc65b7974e3f4a45b5d9df90a5e3a2cb18c5d76 100644 --- a/ews/coordinator/survey_servers/processor_surveys_wrt.py +++ b/ews/coordinator/survey_servers/processor_surveys_wrt.py @@ -112,51 +112,6 @@ def nested_to_flattened(df): for i in RAW_COLUMNS: df[i] = "" - # add new columns - logger.info('Adding new columns') - NEW_COLUMNS = ['imei', - 'sample_size-number_yellowrust_live', - 'sample_size-number_stemrust_live', - 'dead_stemrust_samples_count', - 'samples_collected', - 'sample_size-number_yellowrust_dead', - 'live_leafrust_samples_count', - 'other_crop', - 'live_yellowrust_samples_count', - 'subscriberid', - 'sample_size-using_barcode', - 'start', - 'score_diseases_count', - 'phonenumber', - 'survey_infromation-location-Accuracy', - 'SET-OF-live_yellowrust_samples', - 'SET-OF-score_diseases', - 'meta-instanceID', - 'deviceid', - 'end', - 'samples_type', - 'live_stemrust_samples_count', - 'dead_yellowrust_samples_count', - 'SET-OF-live_leafrust_samples', - 'other_diseases_group-other_diseases', - 'survey_infromation-location-Altitude', - 'SET-OF-dead_stemrust_samples', - 'comment', - 'sample_size-number_leafrust_live', - 'today', - 'SET-OF-dead_yellowrust_samples', - 'username', - 'SET-OF-live_stemrust_samples', - 'sample_size-number_stemrust_dead_dna', - 'SubmissionDate', - 'survey_infromation-location_name', - 'site_information-crop', - 'surveyor_infromation-institution', - 'surveyor_infromation-surveyor_name'] - - for i in NEW_COLUMNS: - df[i] = "" - # add dedicated rust columns, with default values NEW_RUST_COLUMNS = {'SR.Incident':'N','SR.Severity':'N','SR.Reaction':'N', 'SR.Race':'', 'SR.Genotype':'', 'LR.Incident':'N','LR.Severity':'N','LR.Reaction':'N', 'LR.Race':'', 'LR.Genotype':'', @@ -205,35 +160,35 @@ def get_WRT_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat column_parser_dict = { "ObservationID" : 'KEY', "OriginalID" : 'None', - "PublishedLevel" : 'PublishedLevel', - "Origin" : 'Origin', - "Country" : ('parse_cases',(('name_out','surveyor_infromation-country'),('cases', COUNTRY_ABBREVIATIONS_DICT))), - "Latitude" : 'survey_infromation-location-Latitude', - "Longitude" : 'survey_infromation-location-Longitude', - "FieldAreaHa" : ('parse_dtype',(('name_out','site_information-field_area'),('dtype', float))), - "Cultivar" : 'site_information-variety', - "CollectionDate" : ('parse_date',(('name_out','survey_infromation-survey_date'),('fmt_in','%m/%d/%Y'))), + "PublishedLevel" : 'published_level', + "Origin" : 'origin', + "Country" : ('parse_cases',(('name_out','country'),('cases', COUNTRY_ABBREVIATIONS_DICT))), + "Latitude" : 'latitude', + "Longitude" : 'longitude', + "FieldAreaHa" : ('parse_dtype',(('name_out','field_area'),('dtype', float))), + "Cultivar" : 'variety', + "CollectionDate" : ('parse_date',(('name_out','survey_date'),('fmt_in','%m/%d/%Y'))), "UpdateDate" : 'None', - "GrowthStage" : 'site_information-growth_stage', - "SurveySite" : 'site_information-survey_site', + "GrowthStage" : 'growth_stage', + "SurveySite" : 'survey_site', "listDisease" : 'None', - "YR.Severity" : ('parse_cases',(('name_out','yellow_rust-yellowrust_severity'),('cases', cases_severity),('dtype', int))), - "YR.Incident" : ('parse_cases',(('name_out','yellow_rust-yellowrust_incidence'),('cases', cases_incident))), - "YR.Reaction" : 'yellow_rust-yellowrust_host_plant_reaction', + "YR.Severity" : ('parse_cases',(('name_out','severity_yellow_rust'),('cases', cases_severity),('dtype', int))), + "YR.Incident" : ('parse_cases',(('name_out','incidence_yellow_rust'),('cases', cases_incident))), + "YR.Reaction" : 'None', "YR.Race" : 'None', "YR.Genotype" : 'None', - "SR.Severity" : ('parse_cases',(('name_out','stem_rust-Stemrust_severity'),('cases', cases_severity),('dtype', int))), - "SR.Incident" : ('parse_cases',(('name_out','stem_rust-stemrust_incidence'),('cases', cases_incident))), - "SR.Reaction" : 'stem_rust-stemrust_host_plant_reaction', + "SR.Severity" : ('parse_cases',(('name_out','severity_stem_rust'),('cases', cases_severity),('dtype', int))), + "SR.Incident" : ('parse_cases',(('name_out','incidence_stem_rust'),('cases', cases_incident))), + "SR.Reaction" : 'None', "SR.Race" : 'None', "SR.Genotype" : 'None', - "LR.Severity" : ('parse_cases',(('name_out','leaf_rust-leafrust_severity'),('cases', cases_severity),('dtype', int))), - "LR.Incident" : ('parse_cases',(('name_out','leaf_rust-leafrust_incidence'),('cases', cases_incident))), - "LR.Reaction" : 'leaf_rust-leafrust_host_plant_reaction', + "LR.Severity" : ('parse_cases',(('name_out','severity_leaf_rust'),('cases', cases_severity),('dtype', int))), + "LR.Incident" : ('parse_cases',(('name_out','incidence_leaf_rust'),('cases', cases_incident))), + "LR.Reaction" : 'None', "LR.Race" : 'None', "LR.Genotype" : 'None', - "Septoria.Severity" : 'septoria-septoria_severity', - "Septoria.Incident" : 'septoria-septoria_incidence' + "Septoria.Severity" : 'None', + "Septoria.Incident" : 'None' } # perform a pull from the server, and if it fails write a warning message