From 1322f68d48c63baf94185f086cf242b38f6e9326 Mon Sep 17 00:00:00 2001
From: Jake Smith <jws52@cam.ac.uk>
Date: Tue, 5 Jan 2021 09:46:42 +0000
Subject: [PATCH] Clearer logging of survey download.
---
ProcessorComponents.py | 22 ++++++++++++----------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/ProcessorComponents.py b/ProcessorComponents.py
index e5e6064..0d9198a 100644
--- a/ProcessorComponents.py
+++ b/ProcessorComponents.py
@@ -153,7 +153,7 @@ def process_pre_job_epi(input_args):
def process_in_job_survey(jobPath,status,config,component):
logger.info('started process_in_job_survey()')
- logger.debug('Preparing to download from ODK server')
+ logger.debug('Performing download from ODK server')
credentials_filename = config['Survey']['ServerCredentialsFile']
with open(credentials_filename) as credentials_file:
@@ -162,6 +162,8 @@ def process_in_job_survey(jobPath,status,config,component):
ODK_output_path = f"{jobPath}/ExportRawDB"
# get data from ODK server
+ description_short = 'ODK download'
+ description_long = 'survey download from ODK server'
ODK_jar = '/storage/app/EWS/General/EWS-Coordinator/ODK-Briefcase-v1.18.0.jar'
ODK_download = ['java',
'-jar', ODK_jar,
@@ -171,17 +173,17 @@ def process_in_job_survey(jobPath,status,config,component):
'--odk_url', cred['server'],
'--odk_username',cred['user'],
'--odk_password',cred['pass']]
+
ODK_download_success = True
- logger.debug('Downloading from ODK server')
+ logger.debug('Performing ' + description_long)
try:
- subprocess.run(ODK_download,check=True)
+ subprocess_and_log(ODK_download,description_short,description_long,check=True)
+
except subprocess.CalledProcessError as e:
- logger.warning("Failed to download surveys from ODK server", exc_info=True)
status.reset('WARNING')
ODK_download_success = False
- #endJob(status,premature=True)
#TODO: Check it came down cleanly ($serverOutputDir is created whether cleanly or not, so test more explicitly):
@@ -192,7 +194,9 @@ def process_in_job_survey(jobPath,status,config,component):
ODK_csv_filename = 'SurveyData.csv'
if ODK_download_success:
- logger.debug('Preparing to convert format of ODK download to csv')
+ description_short = 'ODK export'
+ description_long = 'format conversion of ODK download to csv'
+ logger.debug(description_long)
ODK_java_to_csv = ['java',
'-jar', ODK_jar,
@@ -202,16 +206,14 @@ def process_in_job_survey(jobPath,status,config,component):
'--export_directory',ODK_csv_path,
'--export_filename',ODK_csv_filename]
- logger.debug('Converting format of ODK download to csv')
+ logger.debug('Performing ' + description_long)
try:
- subprocess.run(ODK_java_to_csv, check=True)
+ subprocess_and_log(ODK_java_to_csv,description_short,description_long,check=True)
except subprocess.CalledProcessError as e:
- logger.warning('Failed to convert ODK survey data to csv', exc_info=True)
status.reset('WARNING')
ODK_download_success = False
- #endJob(status,premature=True)
if not ODK_download_success:
--
GitLab