FAQ | This is a LIVE service | Changelog

Skip to content
Snippets Groups Projects
Commit 879ef2b9 authored by J.W. Smith's avatar J.W. Smith
Browse files

EnvProcessor now configures the call to env suit plotting.

parent b029de94
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
EnvProcessor.py
+ 111
− 4
View file @ 879ef2b9
......@@ -327,7 +327,23 @@ def open_and_check_config(configFile):
return config
def subprocess_and_log(cmd,description_short,description_long,check=True):
def get_only_existing_globs(file_globs,inplace=True):
if inplace:
for i,fg in enumerate(file_globs):
if not glob.glob(fg):
logger.warning(f'This file pattern returns nothing and therefore does not contribute to result:\n{fg}')
file_globs.pop(i)
else:
globs_out = []
for i,fg in enumerate(file_globs):
if not glob.glob(fg):
logger.warning(f'This file pattern returns nothing and therefore does not contribute to result:\n{fg}')
continue
globs_out += [fg]
return globs_out
def subprocess_and_log(cmd,description_short,description_long,check=True,**kwargs):
'''Run a shell command (described by a comma separated list) and send stdout
and stderr to logfile, and raise any exception.'''
......@@ -336,7 +352,8 @@ def subprocess_and_log(cmd,description_short,description_long,check=True):
cmd,
check=check,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT)
stderr = subprocess.STDOUT,
**kwargs)
for line in process.stdout.decode('utf-8').split(r'\n'):
logger.info(f"{description_short} : " + line)
......@@ -449,7 +466,95 @@ def process_in_job(jobPath,config):
return
def process_EWS_plotting_env2_0(jobPath,config):
'''Calls EWS-plotting by configuring the plotting arguments and passing
them to the underlying script.
Returns a list of output directories or files for transfer.'''
logger.info('started process_EWS_plotting_env2_0()')
# initialise environment
plot_path = '/storage/app/EWS/General/EWS-Plotting-Dev/'
env_map = {
'PATH' : os.environ['PATH'],
'PYTHONPATH' : plot_path,
'R_LIBS' : '/home/ewsmanager/R-packages-EWS-plotting-Dev/plotting_rlibs/',
}
input_dir = f"{jobPath}/processed/"
output_dir = f"{jobPath}/plotting/"
Path(output_dir).mkdir(parents=True, exist_ok=True)
regions = config['SubRegionNames']
EWSPlottingOutputGlobs = []
# work on each region
for region in regions:
python_script = config['Environment']['EWS-Plotting'][region]['PythonScript']
run_config = config['Environment']['EWS-Plotting'][region]['RunConfig']
# Note that this runs all disease types available
# TODO: Add logic to determine which diseases to plot
# prepare command
plot_command = [
'/storage/app/EWS/General/EWS-python/runpy_ews_plotting_env.sh',
python_script,
'-stem_r',f"{input_dir}/{region}/StemRust/RIE_value.csv",
'-stripe_r',f"{input_dir}/{region}/StripeRust/RIE_value.csv",
'-leaf_r',f"{input_dir}/{region}/LeafRust/RIE_value.csv",
'--issue_date_arg',config['StartString'],
'--env_suitability_output_dir_arg',output_dir,
'--sys_config_file','/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json',
'--run_config_file',run_config,
'--plot_daily_images','True',
'--plot_hourly_images','False',
'--plot_stem','True',
'--plot_stripe','True',
'--plot_leaf','True']
logger.info(f"Running EWS-Plotting command:\n'{' '.join(plot_command)}'")
description_short = 'EWS-Plotting'
description_long = description_short
subprocess_and_log(plot_command, description_short, description_long,env=env_map)
# gather the output
EWSPlottingOutputDir = f"{output_dir}/{region.lower()}/images/"
#EWSPlottingOutputGlobs += [
# # daily plots
# f"{EWSPlottingOutputDir}Daily/suitability_{region.lower()}_*_rust_daily_20*.png",
# # weekly plots
# f"{EWSPlottingOutputDir}Weekly/suitability_{region.lower()}_*_rust_total_20*.png"]
EWSPlottingOutputGlobs = [f"{EWSPlottingOutputDir}*"]
# check the output
EWSPlottingOutputGlobs = get_only_existing_globs(EWSPlottingOutputGlobs,inplace=False)
# check there is some output from EWS-plotting
if not EWSPlottingOutputGlobs:
logger.error('EWS-Plotting did not produce any output')
raise RuntimeError
# provide list of files for transfer
EWSPlottingOutputs = sorted([file for glob_str in EWSPlottingOutputGlobs for file in glob.glob(glob_str)])
return EWSPlottingOutputs
def process_EWS_plotting(config):
'''Calls EWS-plotting from shell scripts provided in the config dict.
The shell script must be aware of the cases to be plotted.
Returns a list of output directories or files for transfer.
'''
logger.info('started process_EWS_plotting()')
enable_hourly_plots = "false"
......@@ -465,6 +570,8 @@ def process_EWS_plotting(config):
subprocess_and_log(cmd_envsuit,description_short,description_long)
# TODO: Check that output is successful
# TODO: pass output figures back for handling
return []
......@@ -533,7 +640,7 @@ def process_EWS_plotting_epi(config):
EWSPlottingOutputDir = f"{ews_plot_dir}/epi/images/"
EWSPlottingOutputGlobs = [f"{EWSPlottingOutputDir}{config['RegionName'].lower()}_{config['DiseaseName'].lower()}*.png"]
# TODO: local substitute for EpiUtils.get_only_existing_globs(EWSPlottingOutputGlobs)
# TODO: use get_only_existing_globs(EWSPlottingOutputGlobs)
# check there is some output from EWS-plotting
if not EWSPlottingOutputGlobs:
......@@ -647,7 +754,7 @@ def run_Process():
# Run EWS-plotting command
EWSPlottingOutputs = process_EWS_plotting(configjson)
EWSPlottingOutputs = process_EWS_plotting_env2_0(jobPath,configjson)
logger.info('Finished with EWS-Plotting, appending images to list for transfer')
FilesToSend += EWSPlottingOutputs
......
config_ENV_Ethiopia_template.json
+ 9
− 8
View file @ 879ef2b9
......@@ -11,13 +11,14 @@
"ServerKey" : "/storage/app/EWS/General/EWS-Coordinator/ssh_key_willow",
"Environment" : {
"ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/Ethiopia/fromMO/daily_name/",
"InputFileTemplate" : "WR_EnvSuit_Met_Ethiopia_${StartString}"
},
"EWS-Plotting" : {
"ShellScripts" : [
"/storage/app/EWS/General/EWS-Plotting-Dev/python/ethiopia/run_eth_env_plotting_pine.sh"
],
"RunConfig" : "?",
"PythonScript" : "?"
"EWS-Plotting" : {
"ShellScripts" : [
"/storage/app/EWS/General/EWS-Plotting-Dev/python/ethiopia/run_eth_env_plotting_pine.sh"
],
"Ethiopia" : {
"RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/RUN_CONFIG_ETH_PINE.json",
"PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/ethiopia/ews_plotting_env_ethiopia_main.py"
}
}
}
}
\ No newline at end of file
config_ENV_SouthAsia_template.json
+ 14
− 9
View file @ 879ef2b9
......@@ -11,14 +11,19 @@
"ServerKey" : "/storage/app/EWS/General/EWS-Coordinator/ssh_key_willow",
"Environment" : {
"ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/SouthAsia/fromMO/daily_name/",
"InputFileTemplate" : "WR_EnvSuit_Met_SouthAsia_${StartString}"
},
"EWS-Plotting" : {
"ShellScripts" : [
"/storage/app/EWS/General/EWS-Plotting-Dev/python/bangladesh/run_bang_env_plotting_pine.sh",
"/storage/app/EWS/General/EWS-Plotting-Dev/python/nepal/run_nepal_env_plotting_pine.sh"
],
"RunConfig" : "?",
"PythonScript" : "?"
"EWS-Plotting" : {
"ShellScripts" : [
"/storage/app/EWS/General/EWS-Plotting-Dev/python/bangladesh/run_bang_env_plotting_pine.sh",
"/storage/app/EWS/General/EWS-Plotting-Dev/python/nepal/run_nepal_env_plotting_pine.sh"
],
"Bangladesh" : {
"RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/RUN_CONFIG_BANG_PINE.json",
"PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/bangladesh/ews_plotting_env_bang_main.py"
},
"Nepal" : {
"RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/RUN_CONFIG_NEPAL_PINE.json",
"PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/nepal/ews_plotting_env_nepal_main.py"
}
}
}
}
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment