feat: add groups output list

ews/coordinator/processor_surveys.py

@@ -32,7 +32,7 @@ from shutil import copyfile
 
 from numpy import all as np_all
 from numpy import any as np_any
-from pandas import read_csv, concat
+from pandas import read_csv, concat, DataFrame
 
 from ews.coordinator.survey_servers.processor_surveys_odk import get_ODK_form_as_csv
 from ews.coordinator.survey_servers.processor_surveys_odk_sa import get_ODK_SA_form_as_csv
@@ -313,6 +313,9 @@ class ProcessorSurveys(ProcessorBase):
                         logger.debug(f"Removing group {group_name} from list of groups")
                         del groups[group_name]
 
+            # create a list of each group and their content
+            groups_list = DataFrame(columns=['Group','Content','SourcesFile'])
+
             for group_name,group_content in groups.items():
 
                 logger.info(f"Creating survey group {group_name} which includes {group_content}")
@@ -362,6 +365,9 @@ class ProcessorSurveys(ProcessorBase):
 
                 copyfile(sources_path, output_path)
 
+                # add group to the list of groups
+                groups_list = concat([groups_list, DataFrame({'Group':[group_name],'Content':[group_content],'SourcesFile':[output_filename]})])
+
                 # THIS CAN BE REMOVED ONCE THE GROUPS ARE PROPERLY PICKED UP BY THE METOFFICE
                 if (group_name == 'PROD'):
                     logger.debug('Additionally placing copy of PROD result in job directory without group name')
@@ -372,6 +378,12 @@ class ProcessorSurveys(ProcessorBase):
                     logger.debug(f"as {output_path}")
 
                     copyfile(sources_path, output_path)
+            
+            # output groups_list to a file
+            groups_list_filename = f"{job_path}/upload/list_sources_{config['StartString']}.csv"
+            logger.debug(f"Outputting list of groups to {groups_list_filename}")
+            groups_list.to_csv(groups_list_filename, index=False)
+
         else:
             # run python version without grouping surveys