From a14f5c2e891921d1c0fc009bc3e000fc37463fc3 Mon Sep 17 00:00:00 2001
From: Jake Smith <jws52@cam.ac.uk>
Date: Wed, 29 Jun 2022 15:03:55 +0100
Subject: [PATCH] feat: Email config is moved out of this project.
The email config is required to set up the logging before any other
tasks are done in order to maximise coverage. It is expected that the
location of the required config file is defined in run_Processor.sh,
or exported to local environment before calling Processor.py.
---
.gitignore | 2 --
Processor.py | 31 +++++++++++++++++++++----------
run_Processor.sh | 24 ++++++++++++++++++------
3 files changed, 39 insertions(+), 18 deletions(-)
diff --git a/.gitignore b/.gitignore
index c0d2036..7c75169 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,8 +5,6 @@ logs/
*.zip
ODK-Briefcase-v*.jar
Cred-WRSIS-API.json
-Cred_gmail.json
-MO_FTP_Cred.json
ssh_key_willow
ssh_key_willow.pub
historic_calcs
diff --git a/Processor.py b/Processor.py
index 12e7461..cd02b81 100755
--- a/Processor.py
+++ b/Processor.py
@@ -63,12 +63,18 @@ log_path_project = f"{coordinator_path}logs/log.txt"
# job-specific log file will be written here until a job directory exits, when it will be moved there
log_path_default = f"{coordinator_path}logs/log_{nowString}.txt"
-#maintainers = ['jws52@cam.ac.uk','tm689@cam.ac.uk','rs481@cam.ac.uk','willthurston@gmail.com']
-maintainers = ['jws52@cam.ac.uk','tm689@cam.ac.uk','rs481@cam.ac.uk']
+# get the email credentials file path from the environment variables
+assert 'EMAIL_CRED' in os.environ
+email_credential_fn = os.environ['EMAIL_CRED']
+assert os.path.exists(email_credential_fn)
+with open(email_credential_fn,'r') as f:
+ gmail_config = json.load(f)
-with open('/storage/app/EWS/General/EWS-Coordinator/Cred_gmail.json','r') as f:
- gmailConfig = json.load(f)
+# check contents
+required_keys = ['user','pass','host','port','toaddrs']
+for required_key in required_keys:
+ assert required_key in gmail_config
# load config from python dictionary (could be loaded from json file)
# TODO: smtp handler can only use tls, but ssl is more secure. Look into defining/writing a suitable smtp handler
@@ -127,10 +133,10 @@ logConfigDict = {
'handler_buffered_email': {
'class': 'BufferingSMTPHandler.BufferingSMTPHandler',
'level': 'ERROR',
- 'server': (gmailConfig['host'], gmailConfig['port']), # host, port. 465 fsor SSL, 587 for tls
- 'credentials': (gmailConfig['user'], gmailConfig['pass']),
- 'fromaddr': gmailConfig['user'],
- 'toaddrs': maintainers,
+ 'server': (gmail_config['host'], gmail_config['port']), # host, port. 465 fsor SSL, 587 for tls
+ 'credentials': (gmail_config['user'], gmail_config['pass']),
+ 'fromaddr': gmail_config['user'],
+ 'toaddrs': gmail_config['toaddrs'],
'subject': 'ERROR in EWS Processor',
'formatter': 'detailed',
'filters': ['mask_passwords'],
@@ -138,12 +144,17 @@ logConfigDict = {
},
},
'loggers' : {
- script_name : { # this is activated with logging.getLogger('Process.')
+ # this is activated when this script is imported
+ # i.e. with logging.getLogger('Process.')
+ script_name : {
'level' : 'DEBUG',
'handlers' : ['handler_rot_file','handler_file','handler_buffered_email'],
'propagate' : True,
},
- '__main__' : { # this is activated with logging.getLogger(__name__) when name == '__main__'
+ # this is activated when this script is called on the command line
+ # or from a bash script
+ # i.e. with logging.getLogger(__name__) when name == '__main__'
+ '__main__' : {
'level' : 'DEBUG',
'handlers' : ['handler_rot_file','handler_file','handler_buffered_email'],
'propagate' : True,
diff --git a/run_Processor.sh b/run_Processor.sh
index f1c7851..6786e07 100755
--- a/run_Processor.sh
+++ b/run_Processor.sh
@@ -1,15 +1,27 @@
#!/bin/bash
+# directory containing all environment
+envs=/storage/app/EWS/envs/
+
+# directory containing all custom python packages
+bin=/storage/app/EWS/General
+
# provide custom python packages so they can be imported
-flagdir=/storage/app/EWS/General/EWS-python/custom_modules/flagdir/
-epimodel=/storage/app/EWS/General/EWS-EpiModel/
-advisory=/storage/app/EWS/General/EWS-advisory-builder/
-met_extractor=/storage/app/EWS/General/EWS-met_extractor/era5_met_data_extraction/python/
-plotting=/storage/app/EWS/General/EWS-Plotting/
+flagdir=${bin}/EWS-python/custom_modules/flagdir/
+epimodel=${bin}/EWS-EpiModel/
+advisory=${bin}/EWS-advisory-builder/
+met_extractor=${bin}/EWS-met_extractor/era5_met_data_extraction/python/
+plotting=${bin}/EWS-Plotting/
+
export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_extractor:$plotting
+# provide path to email credentials for logging
+
+export EMAIL_CRED=${envs}/Cred_gmail.json
+
# activate conda environment of python modules so they can be imported
-conda_env=/storage/app/EWS/General/EWS-python/py3EWS
+#TODO: Move conda_env from bin to envs
+conda_env=${bin}/EWS-python/py3EWS
source /storage/app/miniconda3/bin/activate ${conda_env}
# get path of this script (to point to files within the same git repo)
--
GitLab