From 208dbbc8175744867fe6a49602e8bdb8631a45c1 Mon Sep 17 00:00:00 2001 From: Tamas Mona <tm689@cam.ac.uk> Date: Tue, 1 Jun 2021 16:37:27 +0100 Subject: [PATCH] Big region run update. Include the dev plotting changes and subregion plotting. --- Cred-WRSIS-API_admin.json | 4 + ProcessorComponents.py | 105 ++++---- WRSIS-API-test.py | 173 +++++++------ ....json => config_EastAfrica_an_ewsdev.json} | 44 +++- ...ve.json => config_EastAfrica_an_live.json} | 44 +++- configs/config_EastAfrica_fc_ewsdev.json | 211 ++++++++++++++++ ...ve.json => config_EastAfrica_fc_live.json} | 123 ++++------ configs/config_Ethiopia_fc_ewsdev.json | 206 ---------------- configs/config_SouthAsia_an_ewsdev.json | 42 ++-- configs/config_SouthAsia_an_live.json | 42 ++-- configs/config_SouthAsia_big_ewsdev.json | 209 ---------------- configs/config_SouthAsia_big_live.json | 229 ------------------ configs/config_SouthAsia_fc_ewsdev.json | 163 ++++++------- configs/config_SouthAsia_fc_live.json | 38 ++- 14 files changed, 647 insertions(+), 986 deletions(-) create mode 100644 Cred-WRSIS-API_admin.json rename configs/{config_Ethiopia_an_ewsdev.json => config_EastAfrica_an_ewsdev.json} (71%) rename configs/{config_Ethiopia_an_live.json => config_EastAfrica_an_live.json} (70%) create mode 100644 configs/config_EastAfrica_fc_ewsdev.json rename configs/{config_Ethiopia_fc_live.json => config_EastAfrica_fc_live.json} (56%) delete mode 100644 configs/config_Ethiopia_fc_ewsdev.json delete mode 100644 configs/config_SouthAsia_big_ewsdev.json delete mode 100644 configs/config_SouthAsia_big_live.json diff --git a/Cred-WRSIS-API_admin.json b/Cred-WRSIS-API_admin.json new file mode 100644 index 0000000..8e12d47 --- /dev/null +++ b/Cred-WRSIS-API_admin.json @@ -0,0 +1,4 @@ +{ + "Username":"Administrator", + "Password":"Administrator@123" +} \ No newline at end of file diff --git a/ProcessorComponents.py b/ProcessorComponents.py index 180854c..f52434f 100644 --- a/ProcessorComponents.py +++ b/ProcessorComponents.py @@ -1009,17 +1009,16 @@ def process_in_job_env2_0(jobPath,status,config,component): # TODO: check that process_met_office_NAME() produced output as expected - regions = config['SubRegionNames'] + region = config['RegionName'] - for region in regions: - logger.info(f"Calling environmental suitability 2.0 for {region} so wait for output to appear") - - pipeline_config = config["Environment"] - try: - esp.run_pipeline(pipeline_config, region, config["StartString"], extracted=False) - except: - logger.exception(f"Some failure when running EnvSuitPipeline.py") - raise + logger.info(f"Calling environmental suitability 2.0 for {region} so wait for output to appear") + + pipeline_config = config["Environment"] + try: + esp.run_pipeline(pipeline_config, region, config["StartString"], extracted=False) + except: + logger.exception(f"Some failure when running EnvSuitPipeline.py") + raise logger.info('Finished running environmental suitability 2.0') @@ -1428,22 +1427,23 @@ def process_EWS_plotting_env2_0(jobPath,config): Returns a list of output files for transfer.''' logger.info('started process_EWS_plotting_env2_0()') - - # initialise environment - input_dir = f"{jobPath}/processed/" - output_dir = f"{jobPath}/plotting/" - - Path(output_dir).mkdir(parents=True, exist_ok=True) - - regions = config['SubRegionNames'] + main_region = config['RegionName'] + + input_dir = f"{jobPath}/processed/{main_region}" + + subregions = config['SubRegionNames'] EWSPlottingOutputGlobs = [] # work on each region - for region in regions: + for region in subregions: + + output_dir = f"{jobPath}/plotting/{region.lower()}" + + Path(output_dir).mkdir(parents=True, exist_ok=True) - plot_path = config['Environment']['EWS-Plotting'][region]['Path'] + plot_path = config['Environment']['EWS-Plotting']['Path'] env_map = { 'PATH' : os.environ['PATH'], 'PYTHONPATH' : plot_path, @@ -1452,32 +1452,31 @@ def process_EWS_plotting_env2_0(jobPath,config): python_script = config['Environment']['EWS-Plotting'][region]['PythonScript'] - run_config = config['Environment']['EWS-Plotting'][region]['RunConfig'] + sys_config = config['Environment']['EWS-Plotting']['SysConfig'] + run_config = config['Environment']['EWS-Plotting']['RunConfig'] + chart_config = config['Environment']['EWS-Plotting'][region]['ChartConfig'] # Note that this runs all disease types available # logic to determine which diseases to plot - diseases_processed = os.listdir(f"{input_dir}/{region}/") + diseases_processed = os.listdir(f"{input_dir}") plot_stem = 'StemRust' in diseases_processed plot_stripe = 'StripeRust' in diseases_processed - plot_leaf = 'StripeRust' in diseases_processed + plot_leaf = 'LeafRust' in diseases_processed # prepare command plot_command = [ '/storage/app/EWS/General/EWS-python/runpy_ews_plotting_env.sh', python_script, - '-stem_r',f"{input_dir}/{region}/StemRust/RIE_value.csv", - '-stripe_r',f"{input_dir}/{region}/StripeRust/RIE_value.csv", - '-leaf_r',f"{input_dir}/{region}/LeafRust/RIE_value.csv", - '--issue_date_arg',config['StartString'], - '--env_suitability_output_dir_arg',output_dir, - '--sys_config_file','/storage/app/EWS/General/EWS-Plotting/python/data/json_config/SYS_CONFIG_PINE.json', + '-stem_r',f"{input_dir}/StemRust/RIE_value.csv", + '-stripe_r',f"{input_dir}/StripeRust/RIE_value.csv", + '-leaf_r',f"{input_dir}/LeafRust/RIE_value.csv", + '--issue_date',config['StartString'], + '--env_suitability_output_dir',output_dir, + '--sys_config_file',sys_config, '--run_config_file',run_config, - '--plot_daily_images','True', - '--plot_hourly_images','False', - '--plot_stem',str(plot_stem), - '--plot_stripe',str(plot_stripe), - '--plot_leaf',str(plot_leaf)] + '--chart_config_file',chart_config, + '--plot_daily_images','True'] logger.info(f"Running EWS-Plotting command:\n'{' '.join(plot_command)}'") @@ -1486,7 +1485,7 @@ def process_EWS_plotting_env2_0(jobPath,config): subprocess_and_log(plot_command, description_short, description_long,env=env_map) # check the output - EWSPlottingOutputDir = f"{output_dir}/{region.lower()}/images/" + EWSPlottingOutputDir = f"{output_dir}/images/" #EWSPlottingOutputGlobs += [ # # daily plots # f"{EWSPlottingOutputDir}Daily/suitability_{region.lower()}_*_rust_daily_20*.png", @@ -1522,15 +1521,18 @@ def process_EWS_plotting_dep(jobPath,config): deposition_path = f"{jobPath}/{deposition_file_name}" - output_dir = f"{jobPath}/plotting/" - - Path(output_dir).mkdir(parents=True, exist_ok=True) + # TODO: get the filename format part from the config + name_file_wildcard = f"{deposition_path}/deposition_srcs_allregions_*.txt" EWSPlottingOutputGlobs = [] for region in regions: - plot_path = config['Deposition']['EWS-Plotting'][region]['Path'] + output_dir = f"{jobPath}/plotting/{region.lower()}" + + Path(output_dir).mkdir(parents=True, exist_ok=True) + + plot_path = config['Deposition']['EWS-Plotting']['Path'] env_map = { 'PATH' : os.environ['PATH'], 'PYTHONPATH' : plot_path, @@ -1539,7 +1541,11 @@ def process_EWS_plotting_dep(jobPath,config): python_script = config['Deposition']['EWS-Plotting'][region]['PythonScript'] - run_config = config['Deposition']['EWS-Plotting'][region]['RunConfig'] + sys_config = config['Deposition']['EWS-Plotting']['SysConfig'] + name_extraction_config = config['Deposition']['EWS-Plotting']['NameExtractionConfig'] + run_config = config['Deposition']['EWS-Plotting']['RunConfig'] + run_config_norm = config['Deposition']['EWS-Plotting']['RunConfigNorm'] + chart_config = config['Deposition']['EWS-Plotting'][region]['ChartConfig'] # Note that this runs all disease types available @@ -1547,13 +1553,16 @@ def process_EWS_plotting_dep(jobPath,config): plot_command = [ '/storage/app/EWS/General/EWS-python/runpy_ews_plotting_env.sh', python_script, - '--issue_date_arg',config['StartString'], - '--deposition_NAME_input_dir',deposition_path, - '--wheat_sources_dir_arg',deposition_path, - '--depo_output_dir_arg',output_dir, - '--sys_config_file','/storage/app/EWS/General/EWS-Plotting/python/data/json_config/SYS_CONFIG_PINE.json', - '--run_config_file',run_config, - '--plot_hourly_images','False'] + '--sys_config_file',sys_config, + '--name_extraction_config_file',name_extraction_config, + '--chart_config_file',chart_config, + '--depo_plotting_run_config_file',run_config, + '--depo_normalized_plotting_run_config_file',run_config_norm, + '--name_file_wildcard',name_file_wildcard, + '--wheat_sources_dir',deposition_path, + '--output_dir',output_dir, + '--issue_date',config['StartString'], + '--plot_daily','True'] logger.info(f"Running EWS-Plotting command:\n'{' '.join(plot_command)}'") @@ -1562,7 +1571,7 @@ def process_EWS_plotting_dep(jobPath,config): subprocess_and_log(plot_command, description_short, description_long,env=env_map) # check the output - EWSPlottingOutputDir = f"{output_dir}/{region.lower()}/images/" + EWSPlottingOutputDir = f"{output_dir}/images/" #EWSPlottingOutputGlobs += [ # # daily plots # f"{EWSPlottingOutputDir}Daily/deposition_{region.lower()}_*_daily_20*.png", diff --git a/WRSIS-API-test.py b/WRSIS-API-test.py index 0b5d198..2e709b6 100755 --- a/WRSIS-API-test.py +++ b/WRSIS-API-test.py @@ -7,76 +7,109 @@ import requests # json is to read the restricted credentials import json -# development-stage request -#gis_url = 'http://197.156.117.171/WRSIS/gisapi/getRustSurveyDataForGlobalRust' -# proposed final url - accesses the 'published' authenticated survey data, so -# waits for quality control -gis_url = 'http://197.156.117.171/WRSIS/gisapi/getUKMetSurveydata' -# TODO: test the final url for raw survey data, yet to be provided by CSM - -cred_fn = 'Cred-WRSIS-API.json' - -# load credentials from restricted file -with open(cred_fn,'r') as cred_file: - cred_dict = json.load(cred_file) - -# The API document says these parameters are mandatory -# example dates as provided in the document -date_params = { - 'fromDate':'10-06-2020', - 'toDate':'22-06-2020'} - -# set up http session -session = requests.Session() - -# provide authorisation -# Surprisingly, this script seems to behave the same if session.auth is not provided -#session.auth = (cred_dict['Username'], cred_dict['Password']) - -# post is a type of HTTP interaction command -# mandatory parameters are passed as the 'json' argument -r = session.post(gis_url,json=date_params) - -# possible HTTP responses as provided in the API document -# (I've seen some other responses though, e.g. 415) -# It seems there is another layer of status codes -status_codes = { - 200 : 'OK', - 201 : 'Created', - 202 : 'Accepted (Request accepted, and queued for execution)', - 400 : 'Bad request', - 401 : 'Authentication failure', - 403 : 'Forbidden', - 404 : 'Resource not found', - 405 : 'Method Not Allowed', - 409 : 'Conflict', - 412 : 'Precondition Failed', - 413 : 'Request Entity Too Large', - 500 : 'Internal Server Error', - 501 : 'Not Implemented', - 503 : 'Service Unavailable'} - -# checking the HTTP status code (not the code in the repsonse) -if r.status_code == 200: - print('HTTP request succeeded OK') - - print('Response is') - # extract the dict from the responses - # r.content is 'bytes' type. Extraction depends on versions of python modules - response = json.loads(str(r.content)[2:-1]) - print(json.dumps(response,indent=2)) - -elif r.status_code in status_codes: - print("HTTP response did not succeed OK, code is {:d}: {:s} ".format(r.status_code,status_codes[r.status_code])) - -else: - print("HTTP response did not succeed OK, unknown code {:d}".format(r.status_code)) - -# TODO: test the sending of GIS data (use the zipped file format returned from CSM) - -gis_send_url = 'http://197.156.117.171/WRSIS/gisapi/uploadGisZipFile' - -# public IP: 134.164.122.165 +def WRSIS_download(server_IP, startDate, endDate): + url = 'http://' + server_IP + '/WRSIS/gisapi/getUKMetSurveyData' + cred_fn = 'Cred-WRSIS-API.json' + date_params = { + 'fromDate':startDate, + 'toDate':endDate} + + # load credentials from restricted file + with open(cred_fn,'r') as cred_file: + cred_dict = json.load(cred_file) + + # set up http session + session = requests.Session() + + # provide authorisation + # Surprisingly, this script seems to behave the same if session.auth is not provided + #session.auth = (cred_dict['Username'], cred_dict['Password']) + + r = session.post(url,json=date_params) + + # possible HTTP responses as provided in the API document + # (I've seen some other responses though, e.g. 415) + # It seems there is another layer of status codes + status_codes = { + 200 : 'OK', + 201 : 'Created', + 202 : 'Accepted (Request accepted, and queued for execution)', + 400 : 'Bad request', + 401 : 'Authentication failure', + 403 : 'Forbidden', + 404 : 'Resource not found', + 405 : 'Method Not Allowed', + 409 : 'Conflict', + 412 : 'Precondition Failed', + 413 : 'Request Entity Too Large', + 500 : 'Internal Server Error', + 501 : 'Not Implemented', + 503 : 'Service Unavailable'} + + # checking the HTTP status code (not the code in the repsonse) + if r.status_code == 200: + print('HTTP request succeeded OK') + + print('Response is') + # extract the dict from the responses + # r.content is 'bytes' type. Extraction depends on versions of python modules + response = json.loads(str(r.content)[2:-1]) + print(json.dumps(response,indent=2)) + + elif r.status_code in status_codes: + print("HTTP response did not succeed OK, code is {:d}: {:s} ".format(r.status_code,status_codes[r.status_code])) + + else: + print("HTTP response did not succeed OK, unknown code {:d}".format(r.status_code)) + +def WRSIS_upload(server_IP, file2upload): + url = 'http://' + server_IP + '/WRSIS/gisapi/uploadGisZipFile' + cred_fn = 'Cred-WRSIS-API_admin.json' + file_param = {'files':file2upload} + + # load credentials from restricted file + with open(cred_fn,'r') as cred_file: + cred_dict = json.load(cred_file) + + # set up http session + session = requests.Session() + + # provide authorisation + # Surprisingly, this script seems to behave the same if session.auth is not provided + #session.auth = (cred_dict['Username'], cred_dict['Password']) + + r = session.post(url,json=file_param) + + # possible HTTP responses as provided in the API document + status_codes = { + 200 : 'OK', + 401 : 'Invalid file', + 500 : 'Unable to upload as server has down . Please try again later.'} + + # checking the HTTP status code (not the code in the repsonse) + if r.status_code == 200: + print('HTTP request succeeded OK') + + print('Response is') + # extract the dict from the responses + # r.content is 'bytes' type. Extraction depends on versions of python modules + response = json.loads(str(r.content)[2:-1]) + print(json.dumps(response,indent=2)) + + elif r.status_code in status_codes: + print("HTTP response did not succeed OK, code is {:d}: {:s} ".format(r.status_code,status_codes[r.status_code])) + + else: + print("HTTP response did not succeed OK, unknown code {:d}".format(r.status_code)) + + +#server_IP = '197.156.117.171' # MoA IP +server_IP = '164.164.122.165:8080' # CSM development IP +#server_IP = '134.164.122.165' # public IP + +WRSIS_download(server_IP,'10-06-2020','22-06-2020') + +WRSIS_upload(server_IP,'forcastingmap.zip') print('Finished script') diff --git a/configs/config_Ethiopia_an_ewsdev.json b/configs/config_EastAfrica_an_ewsdev.json similarity index 71% rename from configs/config_Ethiopia_an_ewsdev.json rename to configs/config_EastAfrica_an_ewsdev.json index ac50e18..e43486a 100644 --- a/configs/config_Ethiopia_an_ewsdev.json +++ b/configs/config_EastAfrica_an_ewsdev.json @@ -1,6 +1,6 @@ { - "RegionName" : "Ethiopia", - "SubRegionNames" : ["Ethiopia"], + "RegionName" : "EastAfrica", + "SubRegionNames" : ["EastAfrica","Ethiopia","Kenya"], "StartTime" : "?", "StartString" : "?", "WorkspacePathout" : "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_Ethiopia_analysis/", @@ -74,25 +74,47 @@ } }, "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/env_suit/RUN_CONFIG_ENV.json", + "EastAfrica" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_EAST_AFRICA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/east_africa/ews_plotting_env_east_africa_main.py" + }, "Ethiopia" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ETH_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/ethiopia/ews_plotting_env_ethiopia_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_ETHIOPIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/ethiopia/ews_plotting_env_ethiopia_main.py" + }, + "Kenya" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_KENYA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/kenya/ews_plotting_env_kenya_main.py" } } }, "Deposition" : { - "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/${RegionName}/fromMO/daily_name/", - "InputFileTemplate" : "WR_NAME_${RegionName}_${StartString}_an", - "TimeExpectedAvailable" : "1400", + "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/Ethiopia/fromMO/daily_name/", + "InputFileTemplate" : "WR_NAME_Ethiopia_${StartString}_fc", + "TimeExpectedAvailable" : "0800", "ProcessPreJob" : "process_pre_job_server_download", "ProcessInJob" : "process_in_job_dep", "ProcessEWSPlotting" : "process_EWS_plotting_dep", "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "NameExtractionConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/name_extraction/NAME_EXTRACTION_CONFIG_LOCAL.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO.json", + "RunConfigNorm" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO_NORMALIZED.json", + "EastAfrica" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_EAST_AFRICA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/east_africa/ews_plotting_depo_east_africa_main.py" + }, "Ethiopia" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ETH_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/ethiopia/ews_plotting_depo_ethiopia_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_ETHIOPIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/ethiopia/ews_plotting_depo_ethiopia_main.py" + }, + "Kenya" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_KENYA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/kenya/ews_plotting_depo_kenya_main.py" } } }, diff --git a/configs/config_Ethiopia_an_live.json b/configs/config_EastAfrica_an_live.json similarity index 70% rename from configs/config_Ethiopia_an_live.json rename to configs/config_EastAfrica_an_live.json index 74109c5..5c08c97 100644 --- a/configs/config_Ethiopia_an_live.json +++ b/configs/config_EastAfrica_an_live.json @@ -1,6 +1,6 @@ { - "RegionName" : "Ethiopia", - "SubRegionNames" : ["Ethiopia"], + "RegionName" : "EastAfrica", + "SubRegionNames" : ["EastAfrica","Ethiopia","Kenya"], "StartTime" : "?", "StartString" : "?", "WorkspacePathout" : "/storage/app/EWS/Ethiopia/Workspace_analysis/", @@ -74,25 +74,47 @@ } }, "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/env_suit/RUN_CONFIG_ENV.json", + "EastAfrica" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_EAST_AFRICA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/east_africa/ews_plotting_env_east_africa_main.py" + }, "Ethiopia" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ETH_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/ethiopia/ews_plotting_env_ethiopia_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_ETHIOPIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/ethiopia/ews_plotting_env_ethiopia_main.py" + }, + "Kenya" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_KENYA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/kenya/ews_plotting_env_kenya_main.py" } } }, "Deposition" : { - "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/${RegionName}/fromMO/daily_name/", - "InputFileTemplate" : "WR_NAME_${RegionName}_${StartString}_an", - "TimeExpectedAvailable" : "1400", + "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/Ethiopia/fromMO/daily_name/", + "InputFileTemplate" : "WR_NAME_Ethiopia_${StartString}_fc", + "TimeExpectedAvailable" : "0800", "ProcessPreJob" : "process_pre_job_server_download", "ProcessInJob" : "process_in_job_dep", "ProcessEWSPlotting" : "process_EWS_plotting_dep", "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "NameExtractionConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/name_extraction/NAME_EXTRACTION_CONFIG_LOCAL.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO.json", + "RunConfigNorm" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO_NORMALIZED.json", + "EastAfrica" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_EAST_AFRICA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/east_africa/ews_plotting_depo_east_africa_main.py" + }, "Ethiopia" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ETH_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/ethiopia/ews_plotting_depo_ethiopia_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_ETHIOPIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/ethiopia/ews_plotting_depo_ethiopia_main.py" + }, + "Kenya" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_KENYA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/kenya/ews_plotting_depo_kenya_main.py" } } }, diff --git a/configs/config_EastAfrica_fc_ewsdev.json b/configs/config_EastAfrica_fc_ewsdev.json new file mode 100644 index 0000000..b197b4c --- /dev/null +++ b/configs/config_EastAfrica_fc_ewsdev.json @@ -0,0 +1,211 @@ +{ + "RegionName" : "EastAfrica", + "SubRegionNames" : ["EastAfrica","Ethiopia","Kenya"], + "StartTime" : "?", + "StartString" : "?", + "WorkspacePathout" : "/storage/home/tm689/EWS-Coordinator-Dev/scratch_EastAfrica/", + "WorkspacePath" : "/storage/home/tm689/EWS-Coordinator-Dev/scratch_EastAfrica/", + "ServerPath" : "/storage/moved/Ethiopia/", + "ServerName" : "willow.csx.cam.ac.uk", + "ServerKey" : "/storage/app/EWS/General/EWS-Coordinator/ssh_key_willow", + "Survey" : { + "ProcessPreJob" : "process_pre_job_survey", + "ProcessInJob" : "process_in_job_survey", + "ProcessEWSPlotting" : "process_EWS_plotting_survey", + "ServerCredentialsFile" : "/storage/app/EWS/Ethiopia/Workspace/Cred-ODK-EIAR.json", + "ServerPathExtra" : "/storage/sftp/metofficeupload/upload/Ethiopia/toMO/" + }, + "Environment" : { + "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/Ethiopia/fromMO/daily_name/", + "InputFileTemplate" : "WR_EnvSuit_Met_Ethiopia_${StartString}_fc", + "TimeExpectedAvailable" : "0800", + "ProcessPreJob" : "process_pre_job_server_download", + "ProcessInJob" : "process_in_job_env2_0", + "ProcessEWSPlotting" : "process_EWS_plotting_env2_0", + "BIN_PATH" : "/storage/app/EWS/General/EWS-met_extractor/", + "TEMPLATE_PATH": "/storage/app/EWS/General/EWS-met_extractor/configs/", + "WORK_PATH": "/storage/home/tm689/EWS-Coordinator-Dev/scratch_EastAfrica/", + "INPUT_PATH": "/storage/home/tm689/EWS-Coordinator-Dev/scratch_EastAfrica/", + "OUTPUT_PATH": "/storage/home/tm689/EWS-Coordinator-Dev/scratch_EastAfrica/", + "RUN_TYPE": "operational", + "EXTRACTION_DAYS": 7, + "FORECAST_DAYS": 6, + "STRAINS": ["LeafRust","StemRust", "StripeRust"], + "PARAMS": { + "LeafRust": { + "suitability_modules": ["semibool_dewperiod"], + "past_steps": 0, + "future_steps": 7, + "thresholds": { + "temperature": [2,15,20,30], + "precipitation": 0, + "relative_humidity": 90 + } + }, + "StemRust": { + "suitability_modules": ["semibool_dewperiod"], + "past_steps": 0, + "future_steps": 7, + "thresholds": { + "temperature": [2,15,24,30], + "precipitation": 0, + "relative_humidity": 90 + } + }, + "StripeRust": { + "suitability_modules": ["v1_dewperiod"], + "past_steps": 0, + "future_steps": 7, + "thresholds": { + "temperature": [2.37,19.8], + "precipitation": 0, + "relative_humidity": 90 + } + }, + "LeafRust_irrigated": { + "suitability_modules": ["semibool_dewperiod"], + "past_steps": 0, + "future_steps": 7, + "thresholds": { + "temperature": [2,15,20,30], + "precipitation": -1, + "relative_humidity": 90 + } + }, + "StemRust_irrigated": { + "suitability_modules": ["semibool_dewperiod"], + "past_steps": 0, + "future_steps": 7, + "thresholds": { + "temperature": [2,15,24,30], + "precipitation": -1, + "relative_humidity": 90 + } + }, + "StripeRust_irrigated": { + "suitability_modules": ["v1_dewperiod"], + "past_steps": 0, + "future_steps": 7, + "thresholds": { + "temperature": [2.37,19.8], + "precipitation": -1, + "relative_humidity": 90 + } + } + }, + "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/env_suit/RUN_CONFIG_ENV.json", + "EastAfrica" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_EAST_AFRICA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/east_africa/ews_plotting_env_east_africa_main.py" + }, + "Ethiopia" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_ETHIOPIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/ethiopia/ews_plotting_env_ethiopia_main.py" + }, + "Kenya" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_KENYA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/kenya/ews_plotting_env_kenya_main.py" + } + } + }, + "Deposition" : { + "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/Ethiopia/fromMO/daily_name/", + "InputFileTemplate" : "WR_NAME_Ethiopia_${StartString}_fc", + "TimeExpectedAvailable" : "0800", + "ProcessPreJob" : "process_pre_job_server_download", + "ProcessInJob" : "process_in_job_dep", + "ProcessEWSPlotting" : "process_EWS_plotting_dep", + "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "NameExtractionConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/name_extraction/NAME_EXTRACTION_CONFIG_LOCAL.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO.json", + "RunConfigNorm" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO_NORMALIZED.json", + "EastAfrica" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_EAST_AFRICA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/east_africa/ews_plotting_depo_east_africa_main.py" + }, + "Ethiopia" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_ETHIOPIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/ethiopia/ews_plotting_depo_ethiopia_main.py" + }, + "Kenya" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_KENYA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/kenya/ews_plotting_depo_kenya_main.py" + } + } + }, + "Epidemiology" : { + "ProcessPreJob" : "process_pre_job_epi", + "ProcessInJob" : "process_in_job_epi", + "ProcessEWSPlotting" : "process_EWS_plotting_epi", + "Host" : { + "MaxFieldsPerCell" : "1000", + "HostRaster" : "/storage/app/EWS/General/EWS-EpiModel/SAsiaInputs/HostData/WheatRaster.tif" + }, + "Deposition" : { + "VariableName" : "P_RECONDITA_DEPOSITION", + "PathTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/WR_NAME_SouthAsia_${StartString}/", + "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/DEPOSITION_SUCCESS.txt", + "FileNameTemplate" : "deposition_srcs_allregions_C1_T${tIndexPlusOneString}_${iTimeString}.txt", + "FileNamePrepared" : "?" + }, + "Environment" : { + "PathTemplate" : "${WorkspacePath}ENVIRONMENT_${StartString}/WR_EnvSuit_SouthAsia_${StartString}/${DiseaseName}Rust_${RegionName}/${StartString}0000/", + "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_${StartString}/ENVIRONMENT_SUCCESS.txt", + "FileNameTemplate" : "RIE_value.csv", + "FileNamePrepared" : "?" + }, + "Epi" : [ + { + "model" : "Env", + "modelArguments" : {}, + "infectionRasterFileName" : "?", + "description": "env. suitability", + "analysis" : { + "vmin" : 0.0e+0, + "vmax" : 1.5e+1, + "subplot_position" : [0,0], + "cmapString" : "CMRmap_r", + "bounds" : [87.9, 92.7, 20.5, 26.7], + "UTMprojection" : 45 + } + },{ + "model" : "log10Dep", + "modelArguments" : {}, + "infectionRasterFileName" : "?", + "description": "log(spore deposition)", + "analysis" : { + "vmin" : 0.0e+0, + "vmax" : 3.0e+2, + "subplot_position" : [0,1], + "cmapString" : "CMRmap_r", + "bounds" : [87.9, 92.7, 20.5, 26.7], + "UTMprojection" : 45 + } + },{ + "model" : "log10DepEnv", + "modelArguments" : {}, + "infectionRasterFileName" : "?", + "description": "EPI: env.suit $\\times$ log(dep.)", + "analysis" : { + "vmin" : 0.0e+0, + "vmax" : 1.5e+1, + "subplot_position" : [0,2], + "cmapString" : "CMRmap_r", + "bounds" : [87.9, 92.7, 20.5, 26.7], + "UTMprojection" : 45 + } + } + ], + "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting/", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ETH_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/ews_plotting_epi_ethiopia_main.py", + "EpiCase" : "log10DepEnv" + } + } +} \ No newline at end of file diff --git a/configs/config_Ethiopia_fc_live.json b/configs/config_EastAfrica_fc_live.json similarity index 56% rename from configs/config_Ethiopia_fc_live.json rename to configs/config_EastAfrica_fc_live.json index 84f2cd0..9172293 100644 --- a/configs/config_Ethiopia_fc_live.json +++ b/configs/config_EastAfrica_fc_live.json @@ -1,6 +1,6 @@ { - "RegionName" : "Ethiopia", - "SubRegionNames" : ["Ethiopia"], + "RegionName" : "EastAfrica", + "SubRegionNames" : ["EastAfrica","Ethiopia","Kenya"], "StartTime" : "?", "StartString" : "?", "WorkspacePathout" : "/storage/app/EWS/Ethiopia/Workspace/", @@ -13,17 +13,7 @@ "ProcessInJob" : "process_in_job_survey", "ProcessEWSPlotting" : "process_EWS_plotting_survey", "ServerCredentialsFile" : "/storage/app/EWS/Ethiopia/Workspace/Cred-ODK-EIAR.json", - "ServerPathExtra" : "/storage/sftp/metofficeupload/upload/Ethiopia/toMO/", - "FormEdits" : { - "wheat_rust_survey_1_0" : {}, - "akpyJHvYxkLKPkxFJnPyTW" : { - "filter": { - "surveyor_infromation-country" : "Kenya" - } - } - }, - "SourcesConfigFilename" : "/storage/app/EWS/General/wheat-source-generation/data/config_EastAfrica_mapspam2017.R", - "SourcesRegionName" : "EastAfrica" + "ServerPathExtra" : "/storage/sftp/metofficeupload/upload/Ethiopia/toMO/" }, "Environment" : { "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/Ethiopia/fromMO/daily_name/", @@ -74,53 +64,71 @@ } }, "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/env_suit/RUN_CONFIG_ENV.json", + "EastAfrica" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_EAST_AFRICA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/east_africa/ews_plotting_env_east_africa_main.py" + }, "Ethiopia" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ETH_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/ethiopia/ews_plotting_env_ethiopia_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_ETHIOPIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/ethiopia/ews_plotting_env_ethiopia_main.py" + }, + "Kenya" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_KENYA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/kenya/ews_plotting_env_kenya_main.py" } } }, "Deposition" : { - "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/${RegionName}/fromMO/daily_name/", - "InputFileTemplate" : "WR_NAME_${RegionName}_${StartString}_fc", + "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/Ethiopia/fromMO/daily_name/", + "InputFileTemplate" : "WR_NAME_Ethiopia_${StartString}_fc", "TimeExpectedAvailable" : "0800", "ProcessPreJob" : "process_pre_job_server_download", "ProcessInJob" : "process_in_job_dep", "ProcessEWSPlotting" : "process_EWS_plotting_dep", "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "NameExtractionConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/name_extraction/NAME_EXTRACTION_CONFIG_LOCAL.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO.json", + "RunConfigNorm" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO_NORMALIZED.json", + "EastAfrica" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_EAST_AFRICA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/east_africa/ews_plotting_depo_east_africa_main.py" + }, "Ethiopia" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ETH_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/ethiopia/ews_plotting_depo_ethiopia_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_ETHIOPIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/ethiopia/ews_plotting_depo_ethiopia_main.py" + }, + "Kenya" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_KENYA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/kenya/ews_plotting_depo_kenya_main.py" } } }, "Epidemiology" : { - "DiseaseNames" : ["StemRust","StripeRust","LeafRust"], - "CalculationSpanDays" : [0,5], - "TimeStep_hours": "3", "ProcessPreJob" : "process_pre_job_epi", "ProcessInJob" : "process_in_job_epi", "ProcessEWSPlotting" : "process_EWS_plotting_epi", + "Host" : { + "MaxFieldsPerCell" : "1000", + "HostRaster" : "/storage/app/EWS/General/EWS-EpiModel/SAsiaInputs/HostData/WheatRaster.tif" + }, "Deposition" : { - "PathTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/WR_NAME_Ethiopia_${StartString}_fc/", - "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS", + "VariableName" : "P_RECONDITA_DEPOSITION", + "PathTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/WR_NAME_SouthAsia_${StartString}/", + "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/DEPOSITION_SUCCESS.txt", "FileNameTemplate" : "deposition_srcs_allregions_C1_T${tIndexPlusOneString}_${iTimeString}.txt", - "FileListerFunction" : "list_deposition_files_operational", "FileNamePrepared" : "?" }, "Environment" : { - "PathTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/processed/${SubRegionName}/${DiseaseName}/", - "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS", + "PathTemplate" : "${WorkspacePath}ENVIRONMENT_${StartString}/WR_EnvSuit_SouthAsia_${StartString}/${DiseaseName}Rust_${RegionName}/${StartString}0000/", + "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_${StartString}/ENVIRONMENT_SUCCESS.txt", "FileNameTemplate" : "RIE_value.csv", - "FileListerFunction" : "list_env_suit_files_operational", "FileNamePrepared" : "?" }, - "Host": { - "MaxFieldsPerCell": "1", - "HostRaster": "/storage/home/jws52/projects/EWS-EpiModel/input_Ethiopia/HOST/wheat_area_frac_MapSPAM2017SSA_Ethiopia.tif" - }, "Epi" : [ { "model" : "Env", @@ -132,8 +140,8 @@ "vmax" : 1.5e+1, "subplot_position" : [0,0], "cmapString" : "CMRmap_r", - "bounds" : [32.5,48.0,3.3,15.0], - "UTMprojection" : 37 + "bounds" : [87.9, 92.7, 20.5, 26.7], + "UTMprojection" : 45 } },{ "model" : "log10Dep", @@ -145,8 +153,8 @@ "vmax" : 3.0e+2, "subplot_position" : [0,1], "cmapString" : "CMRmap_r", - "bounds" : [32.5,48.0,3.3,15.0], - "UTMprojection" : 37 + "bounds" : [87.9, 92.7, 20.5, 26.7], + "UTMprojection" : 45 } },{ "model" : "log10DepEnv", @@ -158,49 +166,16 @@ "vmax" : 1.5e+1, "subplot_position" : [0,2], "cmapString" : "CMRmap_r", - "bounds" : [32.5,48.0,3.3,15.0], - "UTMprojection" : 37 + "bounds" : [87.9, 92.7, 20.5, 26.7], + "UTMprojection" : 45 } } ], "EWS-Plotting" : { "Path" : "/storage/app/EWS/General/EWS-Plotting/", "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ETH_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/ethiopia/ews_plotting_epi_ethiopia_main.py", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/ews_plotting_epi_ethiopia_main.py", "EpiCase" : "log10DepEnv" } - }, - "Advisory" : - { - "ProcessPreJob" : "query_past_successes", - "ProcessInJob" : "process_in_job_advisory", - "ProcessEWSPlotting" : "do_nothing", - "seasonStartString" : "20200601", - "Environment" : { - "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS", - "PlotPathTemplate" : "${WorkspacePath}/ENVIRONMENT_2.0_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/", - "PlotFilenameTemplates" : { - "stripe" : "${envPlotPath}/suitability_${SubRegionNameLower}_stripe_rust_total_${dateString}0000_${EnvEndString}0000_map.png", - "stem" : "${envPlotPath}/suitability_${SubRegionNameLower}_stem_rust_total_${dateString}0000_${EnvEndString}0000_map.png", - "leaf" : "${envPlotPath}/suitability_${SubRegionNameLower}_leaf_rust_total_${dateString}0000_${EnvEndString}0000_map.png" - } - }, - "Deposition" : { - "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS", - "PlotPathTemplate" : "${WorkspacePath}/DEPOSITION_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/", - "PlotFilenameTemplates" : { - "stripe": "${depPlotPath}/deposition_${SubRegionNameLower}_stripe_total_${dateString}0000_${weekAheadString}0000_map.png", - "stem": "${depPlotPath}/deposition_${SubRegionNameLower}_stem_total_${dateString}0000_${weekAheadString}0000_map.png", - "leaf": "${depPlotPath}/deposition_${SubRegionNameLower}_leaf_total_${dateString}0000_${weekAheadString}0000_map.png" - } - }, - "Surveys" : { - "Ethiopia" : { - "ShapeFilenameAdmin0" : "/storage/app/EWS/Ethiopia/Documents/shapefiles/admin-boundaries/Ethiopia_All/Eth_Region_2013.shp", - "ShapeFilenameAdmin1" : "/storage/app/EWS/Ethiopia/Documents/shapefiles/admin-boundaries/Ethiopia_All/Eth_Zone_2013.shp", - "ShapeFilenameAdmin2" : "/storage/app/EWS/Ethiopia/Documents/shapefiles/admin-boundaries/Ethiopia_All/Eth_Woreda_2013.shp" - } - }, - "logos" : "/storage/app/EWS/Ethiopia/Documents/logos_Ethiopia_plotting.png" } } \ No newline at end of file diff --git a/configs/config_Ethiopia_fc_ewsdev.json b/configs/config_Ethiopia_fc_ewsdev.json deleted file mode 100644 index 585cdc8..0000000 --- a/configs/config_Ethiopia_fc_ewsdev.json +++ /dev/null @@ -1,206 +0,0 @@ -{ - "RegionName" : "Ethiopia", - "SubRegionNames" : ["Ethiopia"], - "StartTime" : "?", - "StartString" : "?", - "WorkspacePathout" : "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_Ethiopia/", - "WorkspacePath" : "/storage/app/EWS/Ethiopia/Workspace/", - "ServerPath" : "/storage/home/ewsmanager/scratch_Ethiopia/", - "ServerName" : "willow.csx.cam.ac.uk", - "ServerKey" : "/storage/app/EWS/General/EWS-Coordinator/ssh_key_willow", - "Survey" : { - "ProcessPreJob" : "process_pre_job_survey", - "ProcessInJob" : "process_in_job_survey", - "ProcessEWSPlotting" : "process_EWS_plotting_survey", - "ServerCredentialsFile" : "/storage/app/EWS/Ethiopia/Workspace/Cred-ODK-EIAR.json", - "ServerPathExtra" : "/storage/sftp/metofficeupload/upload/Ethiopia/toMO/", - "FormEdits" : { - "wheat_rust_survey_1_0" : {}, - "akpyJHvYxkLKPkxFJnPyTW" : { - "filter": { - "surveyor_infromation-country" : "Kenya" - } - } - }, - "SourcesConfigFilename" : "/storage/app/EWS/General/wheat-source-generation/data/config_EastAfrica_mapspam2017.R", - "SourcesRegionName" : "EastAfrica" - }, - "Environment" : { - "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/Ethiopia/fromMO/daily_name/", - "InputFileTemplate" : "WR_EnvSuit_Met_Ethiopia_${StartString}_fc", - "TimeExpectedAvailable" : "0800", - "ProcessPreJob" : "process_pre_job_server_download", - "ProcessInJob" : "process_in_job_env2_0", - "ProcessEWSPlotting" : "process_EWS_plotting_env2_0", - "BIN_PATH" : "/storage/app/EWS/General/EWS-met_extractor/", - "TEMPLATE_PATH": "/storage/app/EWS/General/EWS-met_extractor/configs/", - "WORK_PATH": "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_Ethiopia/", - "INPUT_PATH": "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_Ethiopia/", - "OUTPUT_PATH": "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_Ethiopia/", - "RUN_TYPE": "operational", - "EXTRACTION_DAYS": 7, - "FORECAST_DAYS": 6, - "STRAINS": ["LeafRust","StemRust", "StripeRust"], - "PARAMS": { - "LeafRust": { - "suitability_modules": ["semibool_dewperiod"], - "past_steps": 0, - "future_steps": 7, - "thresholds": { - "temperature": [2,15,20,30], - "precipitation": 0, - "relative_humidity": 90 - } - }, - "StemRust": { - "suitability_modules": ["semibool_dewperiod"], - "past_steps": 0, - "future_steps": 7, - "thresholds": { - "temperature": [2,15,24,30], - "precipitation": 0, - "relative_humidity": 90 - } - }, - "StripeRust": { - "suitability_modules": ["v1_dewperiod"], - "past_steps": 0, - "future_steps": 7, - "thresholds": { - "temperature": [2.37,19.8], - "precipitation": 0, - "relative_humidity": 90 - } - } - }, - "EWS-Plotting" : { - "Ethiopia" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ETH_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/ethiopia/ews_plotting_env_ethiopia_main.py" - } - } - }, - "Deposition" : { - "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/${RegionName}/fromMO/daily_name/", - "InputFileTemplate" : "WR_NAME_${RegionName}_${StartString}_fc", - "TimeExpectedAvailable" : "0800", - "ProcessPreJob" : "process_pre_job_server_download", - "ProcessInJob" : "process_in_job_dep", - "ProcessEWSPlotting" : "process_EWS_plotting_dep", - "EWS-Plotting" : { - "Ethiopia" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ETH_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/ethiopia/ews_plotting_depo_ethiopia_main.py" - } - } - }, - "Epidemiology" : { - "DiseaseNames" : ["StemRust","StripeRust","LeafRust"], - "CalculationSpanDays" : [0,5], - "TimeStep_hours": "3", - "ProcessPreJob" : "process_pre_job_epi", - "ProcessInJob" : "process_in_job_epi", - "ProcessEWSPlotting" : "process_EWS_plotting_epi", - "Deposition" : { - "PathTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/WR_NAME_Ethiopia_${StartString}_fc/", - "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS", - "FileNameTemplate" : "deposition_srcs_allregions_C1_T${tIndexPlusOneString}_${iTimeString}.txt", - "FileListerFunction" : "list_deposition_files_operational", - "FileNamePrepared" : "?" - }, - "Environment" : { - "PathTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/processed/${SubRegionName}/${DiseaseName}/", - "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS", - "FileNameTemplate" : "RIE_value.csv", - "FileListerFunction" : "list_env_suit_files_operational", - "FileNamePrepared" : "?" - }, - "Host": { - "MaxFieldsPerCell": "1", - "HostRaster": "/storage/home/jws52/projects/EWS-EpiModel/input_Ethiopia/HOST/wheat_area_frac_MapSPAM2017SSA_Ethiopia.tif" - }, - "Epi" : [ - { - "model" : "Env", - "modelArguments" : {}, - "infectionRasterFileName" : "?", - "description": "env. suitability", - "analysis" : { - "vmin" : 0.0e+0, - "vmax" : 1.5e+1, - "subplot_position" : [0,0], - "cmapString" : "CMRmap_r", - "bounds" : [32.5,48.0,3.3,15.0], - "UTMprojection" : 37 - } - },{ - "model" : "log10Dep", - "modelArguments" : {}, - "infectionRasterFileName" : "?", - "description": "log(spore deposition)", - "analysis" : { - "vmin" : 0.0e+0, - "vmax" : 3.0e+2, - "subplot_position" : [0,1], - "cmapString" : "CMRmap_r", - "bounds" : [32.5,48.0,3.3,15.0], - "UTMprojection" : 37 - } - },{ - "model" : "log10DepEnv", - "modelArguments" : {}, - "infectionRasterFileName" : "?", - "description": "EPI: env.suit $\\times$ log(dep.)", - "analysis" : { - "vmin" : 0.0e+0, - "vmax" : 1.5e+1, - "subplot_position" : [0,2], - "cmapString" : "CMRmap_r", - "bounds" : [32.5,48.0,3.3,15.0], - "UTMprojection" : 37 - } - } - ], - "EWS-Plotting" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ETH_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/ethiopia/ews_plotting_epi_ethiopia_main.py", - "EpiCase" : "log10DepEnv" - } - }, - "Advisory" : - { - "ProcessPreJob" : "query_past_successes", - "ProcessInJob" : "process_in_job_advisory", - "ProcessEWSPlotting" : "do_nothing", - "seasonStartString" : "20200601", - "Environment" : { - "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS", - "PlotPathTemplate" : "${WorkspacePath}/ENVIRONMENT_2.0_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/", - "PlotFilenameTemplates" : { - "stripe" : "${envPlotPath}/suitability_${SubRegionNameLower}_stripe_rust_total_${dateString}0000_${EnvEndString}0000_map.png", - "stem" : "${envPlotPath}/suitability_${SubRegionNameLower}_stem_rust_total_${dateString}0000_${EnvEndString}0000_map.png", - "leaf" : "${envPlotPath}/suitability_${SubRegionNameLower}_leaf_rust_total_${dateString}0000_${EnvEndString}0000_map.png" - } - }, - "Deposition" : { - "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS", - "PlotPathTemplate" : "${WorkspacePath}/DEPOSITION_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/", - "PlotFilenameTemplates" : { - "stripe": "${depPlotPath}/deposition_${SubRegionNameLower}_stripe_total_${dateString}0000_${weekAheadString}0000_map.png", - "stem": "${depPlotPath}/deposition_${SubRegionNameLower}_stem_total_${dateString}0000_${weekAheadString}0000_map.png", - "leaf": "${depPlotPath}/deposition_${SubRegionNameLower}_leaf_total_${dateString}0000_${weekAheadString}0000_map.png" - } - }, - "Surveys" : { - "Ethiopia" : { - "ShapeFilenameAdmin0" : "/storage/app/EWS/Ethiopia/Documents/shapefiles/admin-boundaries/Ethiopia_All/Eth_Region_2013.shp", - "ShapeFilenameAdmin1" : "/storage/app/EWS/Ethiopia/Documents/shapefiles/admin-boundaries/Ethiopia_All/Eth_Zone_2013.shp", - "ShapeFilenameAdmin2" : "/storage/app/EWS/Ethiopia/Documents/shapefiles/admin-boundaries/Ethiopia_All/Eth_Woreda_2013.shp" - } - }, - "logos" : "/storage/app/EWS/Ethiopia/Documents/logos_Ethiopia_plotting.png" - } -} \ No newline at end of file diff --git a/configs/config_SouthAsia_an_ewsdev.json b/configs/config_SouthAsia_an_ewsdev.json index 7d754c5..f940e36 100644 --- a/configs/config_SouthAsia_an_ewsdev.json +++ b/configs/config_SouthAsia_an_ewsdev.json @@ -1,6 +1,6 @@ { "RegionName" : "SouthAsia", - "SubRegionNames" : ["Bangladesh","Nepal"], + "SubRegionNames" : ["SouthAsia","Bangladesh","Nepal"], "StartTime" : "?", "StartString" : "?", "WorkspacePathout" : "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_SouthAsia_analysis/", @@ -73,35 +73,47 @@ } }, "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/env_suit/RUN_CONFIG_ENV.json", + "SouthAsia" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_ASIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/south_asia/ews_plotting_env_asia_main.py" + }, "Bangladesh" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_BANG_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/ews_plotting_env_bang_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_BANGLADESH_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/bangladesh/ews_plotting_env_bang_main.py" }, "Nepal" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_NEPAL_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/nepal/ews_plotting_env_nepal_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_NEPAL_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/nepal/ews_plotting_env_nepal_main.py" } } }, "Deposition" : { "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/SouthAsia/fromMO/daily_name/", - "InputFileTemplate" : "WR_NAME_SouthAsia_${StartString}_an", - "TimeExpectedAvailable" : "1400", + "InputFileTemplate" : "WR_NAME_SouthAsia_${StartString}_fc", + "TimeExpectedAvailable" : "0800", "ProcessPreJob" : "process_pre_job_server_download", "ProcessInJob" : "process_in_job_dep", "ProcessEWSPlotting" : "process_EWS_plotting_dep", "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "NameExtractionConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/name_extraction/NAME_EXTRACTION_CONFIG_LOCAL.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO.json", + "RunConfigNorm" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO_NORMALIZED.json", + "SouthAsia" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_SOUTH_ASIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/south_asia/ews_plotting_depo_south_asia_main.py" + }, "Bangladesh" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_BANG_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/ews_plotting_depo_bang_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_BANGLADESH_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/bangladesh/ews_plotting_depo_bang_main.py" }, "Nepal" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_NEPAL_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/nepal/ews_plotting_depo_nepal_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_NEPAL_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/nepal/ews_plotting_depo_nepal_main.py" } } }, diff --git a/configs/config_SouthAsia_an_live.json b/configs/config_SouthAsia_an_live.json index e4a9a19..1ed914f 100644 --- a/configs/config_SouthAsia_an_live.json +++ b/configs/config_SouthAsia_an_live.json @@ -1,6 +1,6 @@ { "RegionName" : "SouthAsia", - "SubRegionNames" : ["Bangladesh","Nepal"], + "SubRegionNames" : ["SouthAsia","Bangladesh","Nepal"], "StartTime" : "?", "StartString" : "?", "WorkspacePathout" : "/storage/app/EWS/SouthAsia/Workspace_analysis/", @@ -73,35 +73,47 @@ } }, "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/env_suit/RUN_CONFIG_ENV.json", + "SouthAsia" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_ASIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/south_asia/ews_plotting_env_asia_main.py" + }, "Bangladesh" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_BANG_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/ews_plotting_env_bang_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_BANGLADESH_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/bangladesh/ews_plotting_env_bang_main.py" }, "Nepal" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_NEPAL_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/nepal/ews_plotting_env_nepal_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_NEPAL_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/nepal/ews_plotting_env_nepal_main.py" } } }, "Deposition" : { "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/SouthAsia/fromMO/daily_name/", - "InputFileTemplate" : "WR_NAME_SouthAsia_${StartString}_an", - "TimeExpectedAvailable" : "1400", + "InputFileTemplate" : "WR_NAME_SouthAsia_${StartString}_fc", + "TimeExpectedAvailable" : "0800", "ProcessPreJob" : "process_pre_job_server_download", "ProcessInJob" : "process_in_job_dep", "ProcessEWSPlotting" : "process_EWS_plotting_dep", "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "NameExtractionConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/name_extraction/NAME_EXTRACTION_CONFIG_LOCAL.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO.json", + "RunConfigNorm" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO_NORMALIZED.json", + "SouthAsia" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_SOUTH_ASIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/south_asia/ews_plotting_depo_south_asia_main.py" + }, "Bangladesh" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_BANG_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/ews_plotting_depo_bang_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_BANGLADESH_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/bangladesh/ews_plotting_depo_bang_main.py" }, "Nepal" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_NEPAL_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/nepal/ews_plotting_depo_nepal_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_NEPAL_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/nepal/ews_plotting_depo_nepal_main.py" } } }, diff --git a/configs/config_SouthAsia_big_ewsdev.json b/configs/config_SouthAsia_big_ewsdev.json deleted file mode 100644 index 8046bcf..0000000 --- a/configs/config_SouthAsia_big_ewsdev.json +++ /dev/null @@ -1,209 +0,0 @@ -{ - "RegionName" : "SouthAsia", - "SubRegionNames" : ["SouthAsia"], - "StartTime" : "?", - "StartString" : "?", - "WorkspacePathout" : "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_SouthAsia/", - "WorkspacePath" : "/storage/app/EWS/SouthAsia/Workspace/", - "ServerPath" : "/storage/home/ewsmanager/scratch_SouthAsia/", - "ServerName" : "willow.csx.cam.ac.uk", - "ServerKey" : "/storage/app/EWS/General/EWS-Coordinator/ssh_key_willow", - "Survey" : { - "ProcessPreJob" : "process_pre_job_survey", - "ProcessInJob" : "process_in_job_survey", - "ProcessEWSPlotting" : "process_EWS_plotting_survey", - "ServerCredentialsFile" : "/storage/app/EWS/SouthAsia/Workspace/Cred-ODK-CIMMYT-Bangladesh.json", - "ServerPathExtra" : "/storage/sftp/metofficeupload/upload/SouthAsia/toMO/", - "FormEdits" : { - "wheat_rust_blast_survey_1_1" : {}, - "wheat_rust_survey_Afghanistan" : { - "add" : { - "subscriberid" : ""} - } - }, - "SourcesConfigFilename" : "/storage/app/EWS/General/wheat-source-generation/data/config_SouthAsia_mapspam2005.R", - "SourcesRegionName" : "SouthAsia" - }, - "Environment" : { - "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/SouthAsia/fromMO/daily_name/", - "InputFileTemplate" : "WR_EnvSuit_Met_SouthAsia_${StartString}_fc", - "TimeExpectedAvailable" : "0800", - "ProcessPreJob" : "process_pre_job_server_download", - "ProcessInJob" : "process_in_job_env2_0", - "ProcessEWSPlotting" : "process_EWS_plotting_env2_0", - "BIN_PATH" : "/storage/app/EWS/General/EWS-met_extractor/", - "TEMPLATE_PATH": "/storage/app/EWS/General/EWS-met_extractor/configs/", - "WORK_PATH": "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_SouthAsia/", - "INPUT_PATH": "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_SouthAsia/", - "OUTPUT_PATH": "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_SouthAsia/", - "RUN_TYPE": "operational", - "EXTRACTION_DAYS": 7, - "FORECAST_DAYS": 6, - "STRAINS": ["LeafRust","StemRust", "StripeRust"], - "PARAMS": { - "LeafRust": { - "suitability_modules": ["semibool_dewperiod"], - "past_steps": 0, - "future_steps": 7, - "thresholds": { - "temperature": [2,15,20,30], - "precipitation": 0, - "relative_humidity": 90 - } - }, - "StemRust": { - "suitability_modules": ["semibool_dewperiod"], - "past_steps": 0, - "future_steps": 7, - "thresholds": { - "temperature": [2,15,24,30], - "precipitation": 0, - "relative_humidity": 90 - } - }, - "StripeRust": { - "suitability_modules": ["v1_dewperiod"], - "past_steps": 0, - "future_steps": 7, - "thresholds": { - "temperature": [2.37,19.8], - "precipitation": 0, - "relative_humidity": 90 - } - } - }, - "EWS-Plotting" : { - "SouthAsia" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/RUN_CONFIG_ASIA_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/south_asia/ews_plotting_env_asia_main.py" - } - } - }, - "Deposition" : { - "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/SouthAsia/fromMO/daily_name/", - "InputFileTemplate" : "WR_NAME_SouthAsia_${StartString}_fc", - "TimeExpectedAvailable" : "0800", - "ProcessPreJob" : "process_pre_job_server_download", - "ProcessInJob" : "process_in_job_dep", - "ProcessEWSPlotting" : "process_EWS_plotting_dep", - "EWS-Plotting" : { - "SouthAsia" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/RUN_CONFIG_ASIA_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/south_asia/ews_plotting_depo_asia_main.py" - } - } - }, - "Epidemiology" : { - "DiseaseNames" : ["StripeRust"], - "CalculationSpanDays" : [0,5], - "TimeStep_hours": "3", - "ProcessPreJob" : "process_pre_job_epi", - "ProcessInJob" : "process_in_job_epi", - "ProcessEWSPlotting" : "process_EWS_plotting_epi", - "Deposition" : { - "PathTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/WR_NAME_SouthAsia_${StartString}_fc/", - "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS", - "FileNameTemplate" : "deposition_srcs_allregions_C1_T${tIndexPlusOneString}_${iTimeString}.txt", - "FileListerFunction" : "list_deposition_files_operational", - "FileNamePrepared" : "?" - }, - "Environment" : { - "PathTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/processed/${SubRegionName}/${DiseaseName}/", - "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS", - "FileNameTemplate" : "RIE_value.csv", - "FileListerFunction" : "list_env_suit_files_operational", - "FileNamePrepared" : "?" - }, - "Host": { - "MaxFieldsPerCell": "1", - "HostRaster": "/storage/home/jws52/projects/EWS-EpiModel/input_SouthAsia/wheat_area_frac_MapSPAM2005_SouthAsia.tif" - }, - "Epi" : [ - { - "model" : "Env", - "modelArguments" : {}, - "infectionRasterFileName" : "?", - "description": "env. suitability", - "analysis" : { - "vmin" : 0.0e+0, - "vmax" : 1.5e+1, - "subplot_position" : [0,0], - "cmapString" : "CMRmap_r", - "bounds" : [55,110,7.07,37], - "UTMprojection" : 45 - } - },{ - "model" : "log10Dep", - "modelArguments" : {}, - "infectionRasterFileName" : "?", - "description": "log(spore deposition)", - "analysis" : { - "vmin" : 0.0e+0, - "vmax" : 3.0e+2, - "subplot_position" : [0,1], - "cmapString" : "CMRmap_r", - "bounds" : [55,110,7.07,37], - "UTMprojection" : 45 - } - },{ - "model" : "log10DepEnv", - "modelArguments" : {}, - "infectionRasterFileName" : "?", - "description": "EPI: env.suit $\\times$ log(dep.)", - "analysis" : { - "vmin" : 0.0e+0, - "vmax" : 1.5e+1, - "subplot_position" : [0,2], - "cmapString" : "CMRmap_r", - "bounds" : [55,110,7.07,37], - "UTMprojection" : 45 - } - } - ], - "EWS-Plotting" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ASIA_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/south_asia/ews_plotting_epi_asia_main.py", - "EpiCase" : "log10DepEnv" - } - }, - "Advisory" : { - "ProcessPreJob" : "query_past_successes", - "ProcessInJob" : "process_in_job_advisory", - "ProcessEWSPlotting" : "do_nothing", - "seasonStartString" : "20201201", - "Environment" : { - "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS", - "PlotPathTemplate" : "${WorkspacePath}/ENVIRONMENT_2.0_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/", - "PlotFilenameTemplates" : { - "stripe" : "${envPlotPath}/suitability_${SubRegionNameLower}_stripe_rust_total_${dateString}0000_${EnvEndString}0000_map.png", - "stem" : "${envPlotPath}/suitability_${SubRegionNameLower}_stem_rust_total_${dateString}0000_${EnvEndString}0000_map.png", - "leaf" : "${envPlotPath}/suitability_${SubRegionNameLower}_leaf_rust_total_${dateString}0000_${EnvEndString}0000_map.png" - } - }, - "Deposition" : { - "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS", - "PlotPathTemplate" : "${WorkspacePath}/DEPOSITION_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/", - "PlotFilenameTemplates" : { - "stripe": "${depPlotPath}/deposition_${SubRegionNameLower}_stripe_total_${dateString}0000_${weekAheadString}0000_map.png", - "stem": "${depPlotPath}/deposition_${SubRegionNameLower}_stem_total_${dateString}0000_${weekAheadString}0000_map.png", - "leaf": "${depPlotPath}/deposition_${SubRegionNameLower}_leaf_total_${dateString}0000_${weekAheadString}0000_map.png" - } - }, - "Surveys" : { - "Nepal" : { - "ShapeFilenameAdmin0" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Nepal_admin0.shp", - "ShapeFilenameAdmin1" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Nepal_admin1_withwheat_2017-19.shp", - "ShapeFilenameAdmin2" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Nepal_admin2.shp" - }, - "Bangladesh" : { - "ShapeFilenameAdmin0" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Bangladesh_admin0.shp", - "ShapeFilenameAdmin1" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Bangladesh_admin1.shp", - "ShapeFilenameAdmin2" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Bangladesh_admin2.shp" - } - }, - "logos" : "/storage/app/EWS/SouthAsia/Documents/logos_ARRCC.png" - } -} \ No newline at end of file diff --git a/configs/config_SouthAsia_big_live.json b/configs/config_SouthAsia_big_live.json deleted file mode 100644 index 731a063..0000000 --- a/configs/config_SouthAsia_big_live.json +++ /dev/null @@ -1,229 +0,0 @@ -{ - "RegionName" : "SouthAsia", - "SubRegionNames" : ["Bangladesh","Nepal","SouthAsia"], - "StartTime" : "?", - "StartString" : "?", - "WorkspacePathout" : "/storage/app/EWS/SouthAsia/Workspace/", - "WorkspacePath" : "/storage/app/EWS/SouthAsia/Workspace/", - "ServerPath" : "/storage/moved/SouthAsia/", - "ServerName" : "willow.csx.cam.ac.uk", - "ServerKey" : "/storage/app/EWS/General/EWS-Coordinator/ssh_key_willow", - "Survey" : { - "ProcessPreJob" : "process_pre_job_survey", - "ProcessInJob" : "process_in_job_survey", - "ProcessEWSPlotting" : "process_EWS_plotting_survey", - "ServerCredentialsFile" : "/storage/app/EWS/SouthAsia/Workspace/Cred-ODK-CIMMYT-Bangladesh.json", - "ServerPathExtra" : "/storage/sftp/metofficeupload/upload/SouthAsia/toMO/", - "FormEdits" : { - "wheat_rust_blast_survey_1_1" : {}, - "wheat_rust_survey_Afghanistan" : { - "add" : { - "subscriberid" : ""} - } - }, - "SourcesConfigFilename" : "/storage/app/EWS/General/wheat-source-generation/data/config_SouthAsia_mapspam2005.R", - "SourcesRegionName" : "SouthAsia" - }, - "Environment" : { - "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/SouthAsia/fromMO/daily_name/", - "InputFileTemplate" : "WR_EnvSuit_Met_SouthAsia_${StartString}_fc", - "TimeExpectedAvailable" : "0800", - "ProcessPreJob" : "process_pre_job_server_download", - "ProcessInJob" : "process_in_job_env2_0", - "ProcessEWSPlotting" : "process_EWS_plotting_env2_0", - "BIN_PATH" : "/storage/app/EWS/General/EWS-met_extractor/", - "TEMPLATE_PATH": "/storage/app/EWS/General/EWS-met_extractor/configs/", - "WORK_PATH": "/storage/app/EWS/SouthAsia/Workspace/", - "INPUT_PATH": "/storage/app/EWS/SouthAsia/Workspace/", - "OUTPUT_PATH": "/storage/app/EWS/SouthAsia/Workspace/", - "RUN_TYPE": "operational", - "EXTRACTION_DAYS": 7, - "FORECAST_DAYS": 6, - "STRAINS": ["LeafRust","StemRust", "StripeRust"], - "PARAMS": { - "LeafRust": { - "suitability_modules": ["semibool_dewperiod"], - "past_steps": 0, - "future_steps": 7, - "thresholds": { - "temperature": [2,15,20,30], - "precipitation": 0, - "relative_humidity": 90 - } - }, - "StemRust": { - "suitability_modules": ["semibool_dewperiod"], - "past_steps": 0, - "future_steps": 7, - "thresholds": { - "temperature": [2,15,24,30], - "precipitation": 0, - "relative_humidity": 90 - } - }, - "StripeRust": { - "suitability_modules": ["v1_dewperiod"], - "past_steps": 0, - "future_steps": 7, - "thresholds": { - "temperature": [2.37,19.8], - "precipitation": 0, - "relative_humidity": 90 - } - } - }, - "EWS-Plotting" : { - "Bangladesh" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_BANG_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/ews_plotting_env_bang_main.py" - }, - "Nepal" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_NEPAL_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/nepal/ews_plotting_env_nepal_main.py" - }, - "SouthAsia" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ASIA_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/south_asia/ews_plotting_env_asia_main.py" - } - } - }, - "Deposition" : { - "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/SouthAsia/fromMO/daily_name/", - "InputFileTemplate" : "WR_NAME_SouthAsia_${StartString}_fc", - "TimeExpectedAvailable" : "0800", - "ProcessPreJob" : "process_pre_job_server_download", - "ProcessInJob" : "process_in_job_dep", - "ProcessEWSPlotting" : "process_EWS_plotting_dep", - "EWS-Plotting" : { - "Bangladesh" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_BANG_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/ews_plotting_depo_bang_main.py" - }, - "Nepal" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_NEPAL_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/nepal/ews_plotting_depo_nepal_main.py" - }, - "SouthAsia" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ASIA_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/south_asia/ews_plotting_depo_asia_main.py" - } - } - }, - "Epidemiology" : { - "DiseaseNames" : ["StripeRust","LeafRust","StemRust"], - "CalculationSpanDays" : [0,5], - "TimeStep_hours": "3", - "ProcessPreJob" : "process_pre_job_epi", - "ProcessInJob" : "process_in_job_epi", - "ProcessEWSPlotting" : "process_EWS_plotting_epi", - "Deposition" : { - "PathTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/WR_NAME_SouthAsia_${StartString}_fc/", - "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS", - "FileNameTemplate" : "deposition_srcs_allregions_C1_T${tIndexPlusOneString}_${iTimeString}.txt", - "FileListerFunction" : "list_deposition_files_operational", - "FileNamePrepared" : "?" - }, - "Environment" : { - "PathTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/processed/${SubRegionName}/${DiseaseName}/", - "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS", - "FileNameTemplate" : "RIE_value.csv", - "FileListerFunction" : "list_env_suit_files_operational", - "FileNamePrepared" : "?" - }, - "Host": { - "MaxFieldsPerCell": "1", - "HostRaster": "/storage/home/jws52/projects/EWS-EpiModel/input_SouthAsia/wheat_area_frac_MapSPAM2005_SouthAsia.tif" - }, - "Epi" : [ - { - "model" : "Env", - "modelArguments" : {}, - "infectionRasterFileName" : "?", - "description": "env. suitability", - "analysis" : { - "vmin" : 0.0e+0, - "vmax" : 1.5e+1, - "subplot_position" : [0,0], - "cmapString" : "CMRmap_r", - "bounds" : [55,110,7.07,37], - "UTMprojection" : 45 - } - },{ - "model" : "log10Dep", - "modelArguments" : {}, - "infectionRasterFileName" : "?", - "description": "log(spore deposition)", - "analysis" : { - "vmin" : 0.0e+0, - "vmax" : 3.0e+2, - "subplot_position" : [0,1], - "cmapString" : "CMRmap_r", - "bounds" : [55,110,7.07,37], - "UTMprojection" : 45 - } - },{ - "model" : "log10DepEnv", - "modelArguments" : {}, - "infectionRasterFileName" : "?", - "description": "EPI: env.suit $\\times$ log(dep.)", - "analysis" : { - "vmin" : 0.0e+0, - "vmax" : 1.5e+1, - "subplot_position" : [0,2], - "cmapString" : "CMRmap_r", - "bounds" : [55,110,7.07,37], - "UTMprojection" : 45 - } - } - ], - "EWS-Plotting" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ASIA_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/south_asia/ews_plotting_epi_asia_main.py", - "EpiCase" : "log10DepEnv" - } - }, - "Advisory" : { - "ProcessPreJob" : "query_past_successes", - "ProcessInJob" : "process_in_job_advisory", - "ProcessEWSPlotting" : "do_nothing", - "seasonStartString" : "20201201", - "Environment" : { - "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS", - "PlotPathTemplate" : "${WorkspacePath}/ENVIRONMENT_2.0_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/", - "PlotFilenameTemplates" : { - "stripe" : "${envPlotPath}/suitability_${SubRegionNameLower}_stripe_rust_total_${dateString}0000_${EnvEndString}0000_map.png", - "stem" : "${envPlotPath}/suitability_${SubRegionNameLower}_stem_rust_total_${dateString}0000_${EnvEndString}0000_map.png", - "leaf" : "${envPlotPath}/suitability_${SubRegionNameLower}_leaf_rust_total_${dateString}0000_${EnvEndString}0000_map.png" - } - }, - "Deposition" : { - "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS", - "PlotPathTemplate" : "${WorkspacePath}/DEPOSITION_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/", - "PlotFilenameTemplates" : { - "stripe": "${depPlotPath}/deposition_${SubRegionNameLower}_stripe_total_${dateString}0000_${weekAheadString}0000_map.png", - "stem": "${depPlotPath}/deposition_${SubRegionNameLower}_stem_total_${dateString}0000_${weekAheadString}0000_map.png", - "leaf": "${depPlotPath}/deposition_${SubRegionNameLower}_leaf_total_${dateString}0000_${weekAheadString}0000_map.png" - } - }, - "Surveys" : { - "Nepal" : { - "ShapeFilenameAdmin0" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Nepal_admin0.shp", - "ShapeFilenameAdmin1" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Nepal_admin1_withwheat_2017-19.shp", - "ShapeFilenameAdmin2" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Nepal_admin2.shp" - }, - "Bangladesh" : { - "ShapeFilenameAdmin0" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Bangladesh_admin0.shp", - "ShapeFilenameAdmin1" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Bangladesh_admin1.shp", - "ShapeFilenameAdmin2" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Bangladesh_admin2.shp" - } - }, - "logos" : "/storage/app/EWS/SouthAsia/Documents/logos_ARRCC.png" - } -} \ No newline at end of file diff --git a/configs/config_SouthAsia_fc_ewsdev.json b/configs/config_SouthAsia_fc_ewsdev.json index 4e34cb4..377a199 100644 --- a/configs/config_SouthAsia_fc_ewsdev.json +++ b/configs/config_SouthAsia_fc_ewsdev.json @@ -1,40 +1,32 @@ { "RegionName" : "SouthAsia", - "SubRegionNames" : ["Bangladesh","Nepal"], + "SubRegionNames" : ["SouthAsia","Bangladesh","Nepal"], "StartTime" : "?", "StartString" : "?", - "WorkspacePathout" : "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_SouthAsia/", - "WorkspacePath" : "/storage/app/EWS/SouthAsia/Workspace/", - "ServerPath" : "/storage/home/ewsmanager/scratch_SouthAsia/", + "WorkspacePathout" : "/storage/home/tm689/EWS-Coordinator-Dev/scratch_SouthAsia/", + "WorkspacePath" : "/storage/home/tm689/EWS-Coordinator-Dev/scratch_SouthAsia/", + "ServerPath" : "/storage/moved/SouthAsia/", "ServerName" : "willow.csx.cam.ac.uk", "ServerKey" : "/storage/app/EWS/General/EWS-Coordinator/ssh_key_willow", "Survey" : { "ProcessPreJob" : "process_pre_job_survey", "ProcessInJob" : "process_in_job_survey", "ProcessEWSPlotting" : "process_EWS_plotting_survey", - "ServerCredentialsFile" : "/storage/app/EWS/SouthAsia/Workspace/Cred-ODK-CIMMYT-Bangladesh.json", - "FormEdits" : { - "wheat_rust_blast_survey_1_1" : {}, - "wheat_rust_survey_Afghanistan" : { - "add" : { - "subscriberid" : ""} - } - }, - "SourcesConfigFilename" : "/storage/app/EWS/General/wheat-source-generation/data/config_SouthAsia_mapspam2005.R", - "SourcesRegionName" : "SouthAsia" + "ServerCredentialsFile" : "/storage/app/EWS/SouthAsia/Workspace/Cred-ODK-EIAR.json", + "ServerPathExtra" : "/storage/sftp/metofficeupload/upload/SouthAsia/toMO/" }, "Environment" : { "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/SouthAsia/fromMO/daily_name/", "InputFileTemplate" : "WR_EnvSuit_Met_SouthAsia_${StartString}_fc", "TimeExpectedAvailable" : "0800", "ProcessPreJob" : "process_pre_job_server_download", - "ProcessInJob" : "process_in_job_env2_0", + "ProcessInJob" : "process_copy_past_job_env2_0", "ProcessEWSPlotting" : "process_EWS_plotting_env2_0", "BIN_PATH" : "/storage/app/EWS/General/EWS-met_extractor/", "TEMPLATE_PATH": "/storage/app/EWS/General/EWS-met_extractor/configs/", - "WORK_PATH": "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_SouthAsia/", - "INPUT_PATH": "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_SouthAsia/", - "OUTPUT_PATH": "/storage/app/EWS/General/EWS-Coordinator-Dev/scratch_SouthAsia/", + "WORK_PATH": "/storage/home/tm689/EWS-Coordinator-Dev/scratch_SouthAsia/", + "INPUT_PATH": "/storage/home/tm689/EWS-Coordinator-Dev/scratch_SouthAsia/", + "OUTPUT_PATH": "/storage/home/tm689/EWS-Coordinator-Dev/scratch_SouthAsia/", "RUN_TYPE": "operational", "EXTRACTION_DAYS": 7, "FORECAST_DAYS": 6, @@ -69,18 +61,53 @@ "precipitation": 0, "relative_humidity": 90 } + }, + "LeafRust_irrigated": { + "suitability_modules": ["semibool_dewperiod"], + "past_steps": 0, + "future_steps": 7, + "thresholds": { + "temperature": [2,15,20,30], + "precipitation": -1, + "relative_humidity": 90 + } + }, + "StemRust_irrigated": { + "suitability_modules": ["semibool_dewperiod"], + "past_steps": 0, + "future_steps": 7, + "thresholds": { + "temperature": [2,15,24,30], + "precipitation": -1, + "relative_humidity": 90 + } + }, + "StripeRust_irrigated": { + "suitability_modules": ["v1_dewperiod"], + "past_steps": 0, + "future_steps": 7, + "thresholds": { + "temperature": [2.37,19.8], + "precipitation": -1, + "relative_humidity": 90 + } } }, "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/env_suit/RUN_CONFIG_ENV.json", + "SouthAsia" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_ASIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/south_asia/ews_plotting_env_asia_main.py" + }, "Bangladesh" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_BANG_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/ews_plotting_env_bang_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_BANGLADESH_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/bangladesh/ews_plotting_env_bang_main.py" }, "Nepal" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_NEPAL_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/nepal/ews_plotting_env_nepal_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_NEPAL_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/nepal/ews_plotting_env_nepal_main.py" } } }, @@ -92,43 +119,46 @@ "ProcessInJob" : "process_in_job_dep", "ProcessEWSPlotting" : "process_EWS_plotting_dep", "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "NameExtractionConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/name_extraction/NAME_EXTRACTION_CONFIG_LOCAL.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO.json", + "RunConfigNorm" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO_NORMALIZED.json", + "SouthAsia" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_SOUTH_ASIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/south_asia/ews_plotting_depo_south_asia_main.py" + }, "Bangladesh" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_BANG_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/ews_plotting_depo_bang_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_BANGLADESH_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/bangladesh/ews_plotting_depo_bang_main.py" }, "Nepal" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_NEPAL_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/nepal/ews_plotting_depo_nepal_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_NEPAL_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/nepal/ews_plotting_depo_nepal_main.py" } } }, "Epidemiology" : { - "DiseaseNames" : ["StemRust","StripeRust","LeafRust"], - "CalculationSpanDays" : [0,5], - "TimeStep_hours": "3", "ProcessPreJob" : "process_pre_job_epi", "ProcessInJob" : "process_in_job_epi", "ProcessEWSPlotting" : "process_EWS_plotting_epi", + "Host" : { + "MaxFieldsPerCell" : "1000", + "HostRaster" : "/storage/app/EWS/General/EWS-EpiModel/SAsiaInputs/HostData/WheatRaster.tif" + }, "Deposition" : { - "PathTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/WR_NAME_SouthAsia_${StartString}_fc/", - "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS", + "VariableName" : "P_RECONDITA_DEPOSITION", + "PathTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/WR_NAME_SouthAsia_${StartString}/", + "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/DEPOSITION_SUCCESS.txt", "FileNameTemplate" : "deposition_srcs_allregions_C1_T${tIndexPlusOneString}_${iTimeString}.txt", - "FileListerFunction" : "list_deposition_files_operational", "FileNamePrepared" : "?" }, "Environment" : { - "PathTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/processed/${SubRegionName}/${DiseaseName}/", - "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS", + "PathTemplate" : "${WorkspacePath}ENVIRONMENT_${StartString}/WR_EnvSuit_SouthAsia_${StartString}/${DiseaseName}Rust_${RegionName}/${StartString}0000/", + "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_${StartString}/ENVIRONMENT_SUCCESS.txt", "FileNameTemplate" : "RIE_value.csv", - "FileListerFunction" : "list_env_suit_files_operational", "FileNamePrepared" : "?" }, - "Host": { - "MaxFieldsPerCell": "1", - "HostRaster": "/storage/home/jws52/projects/EWS-EpiModel/input_SouthAsia/wheat_area_frac_MapSPAM2005_SouthAsia.tif" - }, "Epi" : [ { "model" : "Env", @@ -140,7 +170,7 @@ "vmax" : 1.5e+1, "subplot_position" : [0,0], "cmapString" : "CMRmap_r", - "bounds" : [55,110,7.07,37], + "bounds" : [87.9, 92.7, 20.5, 26.7], "UTMprojection" : 45 } },{ @@ -153,7 +183,7 @@ "vmax" : 3.0e+2, "subplot_position" : [0,1], "cmapString" : "CMRmap_r", - "bounds" : [55,110,7.07,37], + "bounds" : [87.9, 92.7, 20.5, 26.7], "UTMprojection" : 45 } },{ @@ -166,53 +196,16 @@ "vmax" : 1.5e+1, "subplot_position" : [0,2], "cmapString" : "CMRmap_r", - "bounds" : [55,110,7.07,37], + "bounds" : [87.9, 92.7, 20.5, 26.7], "UTMprojection" : 45 } } ], "EWS-Plotting" : { "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ASIA_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/south_asia/ews_plotting_epi_asia_main.py", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_ETH_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/ews_plotting_epi_ethiopia_main.py", "EpiCase" : "log10DepEnv" } - }, - "Advisory" : { - "ProcessPreJob" : "query_past_successes", - "ProcessInJob" : "process_in_job_advisory", - "ProcessEWSPlotting" : "do_nothing", - "seasonStartString" : "20201201", - "Environment" : { - "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS", - "PlotPathTemplate" : "${WorkspacePath}/ENVIRONMENT_2.0_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/", - "PlotFilenameTemplates" : { - "stripe" : "${envPlotPath}/suitability_${SubRegionNameLower}_stripe_rust_total_${dateString}0000_${EnvEndString}0000_map.png", - "stem" : "${envPlotPath}/suitability_${SubRegionNameLower}_stem_rust_total_${dateString}0000_${EnvEndString}0000_map.png", - "leaf" : "${envPlotPath}/suitability_${SubRegionNameLower}_leaf_rust_total_${dateString}0000_${EnvEndString}0000_map.png" - } - }, - "Deposition" : { - "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS", - "PlotPathTemplate" : "${WorkspacePath}/DEPOSITION_${dateString}/plotting/${SubRegionNameLower}/images/Weekly/", - "PlotFilenameTemplates" : { - "stripe": "${depPlotPath}/deposition_${SubRegionNameLower}_stripe_total_${dateString}0000_${weekAheadString}0000_map.png", - "stem": "${depPlotPath}/deposition_${SubRegionNameLower}_stem_total_${dateString}0000_${weekAheadString}0000_map.png", - "leaf": "${depPlotPath}/deposition_${SubRegionNameLower}_leaf_total_${dateString}0000_${weekAheadString}0000_map.png" - } - }, - "Surveys" : { - "Nepal" : { - "ShapeFilenameAdmin0" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Nepal_admin0.shp", - "ShapeFilenameAdmin1" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Nepal_admin1_withwheat_2017-19.shp", - "ShapeFilenameAdmin2" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Nepal_admin2.shp" - }, - "Bangladesh" : { - "ShapeFilenameAdmin0" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Bangladesh_admin0.shp", - "ShapeFilenameAdmin1" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Bangladesh_admin1.shp", - "ShapeFilenameAdmin2" : "/storage/app/EWS/SouthAsia/Documents/shapefiles/Bangladesh_admin2.shp" - } - }, - "logos" : "/storage/app/EWS/SouthAsia/Documents/logos_ARRCC.png" } } \ No newline at end of file diff --git a/configs/config_SouthAsia_fc_live.json b/configs/config_SouthAsia_fc_live.json index 5a3da90..615e8b7 100644 --- a/configs/config_SouthAsia_fc_live.json +++ b/configs/config_SouthAsia_fc_live.json @@ -1,6 +1,6 @@ { "RegionName" : "SouthAsia", - "SubRegionNames" : ["Bangladesh","Nepal"], + "SubRegionNames" : ["SouthAsia","Bangladesh","Nepal"], "StartTime" : "?", "StartString" : "?", "WorkspacePathout" : "/storage/app/EWS/SouthAsia/Workspace/", @@ -73,15 +73,20 @@ } }, "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/env_suit/RUN_CONFIG_ENV.json", + "SouthAsia" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_ASIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/south_asia/ews_plotting_env_asia_main.py" + }, "Bangladesh" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_BANG_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/ews_plotting_env_bang_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_BANGLADESH_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/bangladesh/ews_plotting_env_bang_main.py" }, "Nepal" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_NEPAL_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/nepal/ews_plotting_env_nepal_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_NEPAL_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/nepal/ews_plotting_env_nepal_main.py" } } }, @@ -93,15 +98,22 @@ "ProcessInJob" : "process_in_job_dep", "ProcessEWSPlotting" : "process_EWS_plotting_dep", "EWS-Plotting" : { + "Path" : "/storage/app/EWS/General/EWS-Plotting-Dev/", + "SysConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/SYS_CONFIG_PINE.json", + "NameExtractionConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/name_extraction/NAME_EXTRACTION_CONFIG_LOCAL.json", + "RunConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO.json", + "RunConfigNorm" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/deposition/RUN_CONFIG_DEPO_NORMALIZED.json", + "SouthAsia" : { + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_SOUTH_ASIA_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/south_asia/ews_plotting_depo_south_asia_main.py" + }, "Bangladesh" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_BANG_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/ews_plotting_depo_bang_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_BANGLADESH_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/bangladesh/ews_plotting_depo_bang_main.py" }, "Nepal" : { - "Path" : "/storage/app/EWS/General/EWS-Plotting/", - "RunConfig" : "/storage/app/EWS/General/EWS-Plotting/python/data/json_config/RUN_CONFIG_NEPAL_PINE.json", - "PythonScript" : "/storage/app/EWS/General/EWS-Plotting/python/nepal/ews_plotting_depo_nepal_main.py" + "ChartConfig" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/data/json_config/ews/chart/CHART_CONFIG_NEPAL_PINE.json", + "PythonScript" : "/storage/app/EWS/General/EWS-Plotting-Dev/python/chart_areas/nepal/ews_plotting_depo_nepal_main.py" } } }, -- GitLab