From 03604f80c26c068f8c0133ab95337d2705955b96 Mon Sep 17 00:00:00 2001
From: tm689 <tm689@cam.ac.uk>
Date: Mon, 30 Sep 2024 11:49:11 +0100
Subject: [PATCH] feat: add groups output list
---
ews/coordinator/processor_surveys.py | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/ews/coordinator/processor_surveys.py b/ews/coordinator/processor_surveys.py
index d778694..9c875fb 100644
--- a/ews/coordinator/processor_surveys.py
+++ b/ews/coordinator/processor_surveys.py
@@ -32,7 +32,7 @@ from shutil import copyfile
from numpy import all as np_all
from numpy import any as np_any
-from pandas import read_csv, concat
+from pandas import read_csv, concat, DataFrame
from ews.coordinator.survey_servers.processor_surveys_odk import get_ODK_form_as_csv
from ews.coordinator.survey_servers.processor_surveys_odk_sa import get_ODK_SA_form_as_csv
@@ -313,6 +313,9 @@ class ProcessorSurveys(ProcessorBase):
logger.debug(f"Removing group {group_name} from list of groups")
del groups[group_name]
+ # create a list of each group and their content
+ groups_list = DataFrame(columns=['Group','Content','SourcesFile'])
+
for group_name,group_content in groups.items():
logger.info(f"Creating survey group {group_name} which includes {group_content}")
@@ -362,6 +365,9 @@ class ProcessorSurveys(ProcessorBase):
copyfile(sources_path, output_path)
+ # add group to the list of groups
+ groups_list = concat([groups_list, DataFrame({'Group':[group_name],'Content':[group_content],'SourcesFile':[output_filename]})])
+
# THIS CAN BE REMOVED ONCE THE GROUPS ARE PROPERLY PICKED UP BY THE METOFFICE
if (group_name == 'PROD'):
logger.debug('Additionally placing copy of PROD result in job directory without group name')
@@ -372,6 +378,12 @@ class ProcessorSurveys(ProcessorBase):
logger.debug(f"as {output_path}")
copyfile(sources_path, output_path)
+
+ # output groups_list to a file
+ groups_list_filename = f"{job_path}/upload/list_sources_{config['StartString']}.csv"
+ logger.debug(f"Outputting list of groups to {groups_list_filename}")
+ groups_list.to_csv(groups_list_filename, index=False)
+
else:
# run python version without grouping surveys
--
GitLab