Merge remote-tracking branch 'origin/main' into main

coordinator/ProcessorSurveysWRSIS.py

@@ -104,6 +104,7 @@ def nested_to_flattened(df):
                 'Rust Details',
                 'Other Disease',
                 'Sample Details',
+                'Survey Details.Survey No',
                 'Survey Details.Latitude',
                 'Survey Details.First Rust Observation Date',
                 'Survey Details.Longitude',
@@ -163,7 +164,6 @@ def nested_to_flattened(df):
             'live_stemrust_samples_count',
             'dead_yellowrust_samples_count',
             'SET-OF-live_leafrust_samples',
-            'KEY',
             'other_diseases_group-other_diseases',
             'survey_infromation-location-Altitude',
             'SET-OF-dead_stemrust_samples',
@@ -178,9 +178,6 @@ def nested_to_flattened(df):
     for i in NEW_COLUMNS:
         df[i] = ""
 
-    #TODO: replace with a better KEY column
-    df["KEY"] = df.index
-
     # add dedicated rust columns, with default values
     NEW_RUST_COLUMNS = {"Stem Rust.Incident":"none","Stem Rust.Severity":"-9","Stem Rust.Reaction":"na",
                    "Leaf Rust.Incident":"none","Leaf Rust.Severity":"-9","Leaf Rust.Reaction":"na",
@@ -221,6 +218,7 @@ def get_WRSIS_form_as_csv(form_credentials: dict, jobPath: str, config: dict, st
         'Rust Details' : 'None',
         'Other Disease' : 'None',
         'Sample Details' : 'None',
+        'Survey Details.Survey No' : 'KEY',
         'Survey Details.Latitude' : 'survey_infromation-location-Latitude',
         'Survey Details.First Rust Observation Date' : 'None',
         'Survey Details.Longitude' : 'survey_infromation-location-Longitude',

coordinator/ProcessorSurveysWRT.py

@@ -114,7 +114,6 @@ def nested_to_flattened(df):
             'live_stemrust_samples_count',
             'dead_yellowrust_samples_count',
             'SET-OF-live_leafrust_samples',
-            'KEY',
             'other_diseases_group-other_diseases',
             'survey_infromation-location-Altitude',
             'SET-OF-dead_stemrust_samples',
@@ -234,7 +233,11 @@ def get_WRT_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat
         logger.debug('Saving raw csv file')
 
         df_raw_filename = f"{output_path}/{csv_filename}"
-        dataframe_raw = json_normalize(request.json()["listObservation"])
+        # if request is not empty, than convert it to a normalized dataframe, otherwise create an empty dataframe
+        if request.status_code == 200:
+            dataframe_raw = json_normalize(request.json()["listObservation"])
+        if request.status_code == 204:
+            dataframe_raw = json_normalize([])
 
         dataframe_raw.to_csv(df_raw_filename,index=False,quoting=csv.QUOTE_MINIMAL)