From 0d0549edf5b5a48fd3da7730c3e68c98dbb03825 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Mon, 10 Jun 2024 16:13:48 +0100
Subject: [PATCH] first commit in move to get config jsons into gitlab and
parameterize paths
---
.gitlab-ci.yml | 2 +-
ews/coordinator/processor_advisory.py | 2 +-
ews/coordinator/processor_base.py | 138 +++++++++---------
ews/coordinator/processor_deposition.py | 26 ++--
ews/coordinator/processor_environment.py | 12 +-
ews/coordinator/processor_epidemiology.py | 47 +++---
ews/coordinator/processor_surveys.py | 38 ++---
.../processor_surveys_kobotoolbox.py | 4 +-
.../processor_surveys_new_odk.py | 4 +-
.../processor_surveys_new_odk2.py | 4 +-
.../survey_servers/processor_surveys_odk.py | 6 +-
.../processor_surveys_odk_sa.py | 6 +-
.../processor_surveys_odk_v2.py | 4 +-
.../survey_servers/processor_surveys_wrsis.py | 6 +-
.../survey_servers/processor_surveys_wrt.py | 6 +-
ews/coordinator/utils/processor_utils.py | 21 ++-
tests/integration/full/full_test_advisory.py | 4 +-
.../integration/full/full_test_deposition.py | 21 ++-
tests/integration/full/full_test_env_suit.py | 4 +-
tests/integration/full/full_test_epi.py | 4 +-
tests/integration/full/full_test_survey.py | 4 +-
.../partial/integration_test_utils.py | 17 ++-
tests/integration/partial/test_advisory.py | 23 +--
tests/integration/partial/test_deposition.py | 27 ++--
tests/integration/partial/test_env_suit.py | 33 +++--
tests/integration/partial/test_epi.py | 26 ++--
tests/integration/partial/test_survey.py | 28 ++--
.../depo_config_EastAfrica_fc_live.json | 23 +++
.../env_suit_config_EastAfrica_fc_live.json | 92 ++++++++++++
.../epi_config_EastAfrica_fc_live.json | 90 ++++++++++++
.../survey_config_EastAfrica_fc_live.json | 40 +++++
.../sys_config_EastAfrica_fc_live.json | 9 ++
32 files changed, 542 insertions(+), 229 deletions(-)
create mode 100644 tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/depo_config_EastAfrica_fc_live.json
create mode 100644 tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/env_suit_config_EastAfrica_fc_live.json
create mode 100644 tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/epi_config_EastAfrica_fc_live.json
create mode 100644 tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/survey_config_EastAfrica_fc_live.json
create mode 100644 tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/sys_config_EastAfrica_fc_live.json
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index feb1106..7850d83 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -33,7 +33,7 @@ stages:
.before_script:
before_script:
- - BRANCH="main"
+ - BRANCH="lb584_060624"
- echo "branch - " $BRANCH
- PACKAGES_DIR="$CI_PROJECT_DIR/packages/"
- echo "project dir - " $CI_PROJECT_DIR
diff --git a/ews/coordinator/processor_advisory.py b/ews/coordinator/processor_advisory.py
index c6fcd9b..c2e35f2 100644
--- a/ews/coordinator/processor_advisory.py
+++ b/ews/coordinator/processor_advisory.py
@@ -37,7 +37,7 @@ class ProcessorAdvisory(ProcessorBase):
who edits the content of the document.
Uses the gitlab project EWS-advisory-builder.'''
- config_advisory = config[component].copy()
+ config_advisory = config.copy()
config_advisory['jobPath'] = jobPath
diff --git a/ews/coordinator/processor_base.py b/ews/coordinator/processor_base.py
index cb558f3..59a28ec 100755
--- a/ews/coordinator/processor_base.py
+++ b/ews/coordinator/processor_base.py
@@ -21,8 +21,8 @@ from typing import List
from ews.coordinator.utils import processor_utils
from ews.coordinator.utils.jobstatus import Jobstatus
-from ews.coordinator.utils.processor_utils import open_and_check_config, end_script, end_job, append_item_to_list, \
- clear_up
+from ews.coordinator.utils.processor_utils import end_script, end_job, append_item_to_list, \
+ clear_up, parse_json_file_with_tokens
print("Make sure to `conda activate py3EWSepi` environment!")
@@ -62,14 +62,14 @@ class ProcessorBase:
# positional arguments do not start with - or -- and are always required
# optional arguments start with - or -- and default is required = False
- # my_parser.add_argument(
- # '-p', '--component',
- # type = str,
- # choices = list(short_name.keys()),
- # required = False,
- # dest = 'component',
- # help = '''Name of EWS component to process, which must be present
- # in the config file.''')
+ my_parser.add_argument(
+ '-sc', '--sys_config',
+ metavar = 'path',
+ type = str,
+ dest = 'sys_config_path',
+ required = True,
+ help = '''path to the system config file. This file contains values which are common to all Processor
+ components''')
my_parser.add_argument(
'-c', '--config',
@@ -78,9 +78,6 @@ class ProcessorBase:
nargs = '+', # require at least one path
dest = 'config_paths',
required = True,
- # default = ['config_Ethiopia_template_stripe.json'], # remove once live
- # default = ['config_Bangladesh_template_stripe.json'], # remove once live
- # default = ['config_Nepal_template_stripe.json'], # remove once live
help = '''path to a config file(s). More than one can be provided,
in which case each is worked on in turn (e.g. one each for stripe, stem, leaf).
Do not place other options between these.''')
@@ -154,50 +151,47 @@ class ProcessorBase:
return dictionary
- def build_universal_config(self, configs: list, component: str, universal_config = None) -> dict:
- '''This config obtains aspects of each dict in configs that must be common to
- them all. '''
-
- # initialise universal config
- if not universal_config:
- universal_config = {
- 'WorkspacePathout': set()
- }
-
- keys = universal_config.keys()
-
- # get value of each key from each config file
- for configFile in configs:
-
- try:
-
- config_i = open_and_check_config(configFile)
-
- except:
-
- logger.exception(f"Failure in opening or checking config {configFile}")
- end_script(premature = True)
-
- for key in keys:
-
- try:
- universal_config[key].add(config_i[key])
-
- except KeyError:
-
- # key must be in component sub-dict
- universal_config[key].add(config_i[component][key])
-
- # Check for and keep only one value per key
- for key in keys:
-
- if len(universal_config[key]) > 1:
- logger.error(f"Config files point to multiple {key} but this script can only handle one.")
- end_script(premature = True)
-
- universal_config[key] = universal_config[key].pop()
-
- return universal_config
+ # def build_sys_config(self, sys_config_path: str) -> dict:
+ # '''This config obtains aspects of each dict in configs that must be common to
+ # them all. '''
+ #
+ # # initialise universal config
+ # # sys_config = {
+ # # "ProjectRoot": set(),
+ # # 'WorkspacePathout': set(),
+ # # "MetoFTP": set(),
+ # # "WorkspacePath": set(),
+ # # "ResourcesPath": set()
+ # # }
+ #
+ # sys_config = {}
+ # # keys = sys_config.keys()
+ #
+ # # get value of each key from each config file
+ # try:
+ # sys_config = open_and_check_config(sys_config_path)
+ # # for key in keys:
+ # # if key in config_i.keys():
+ # # sys_config[key].add(config_i[key])
+ # except:
+ # logger.exception(f"Failure in opening or checking config {sys_config_path}")
+ # end_script(premature = True)
+ #
+ # # Check for and keep only one value per key
+ # # for key in keys:
+ # # if len(sys_config[key]) > 1:
+ # # logger.error(f"Config files point to multiple {key} but this script can only handle one.")
+ # # end_script(premature = True)
+ # #
+ # # sys_config[key] = sys_config[key].pop()
+ #
+ # # config_tokens = {'ProjectRoot': sys_config['ProjectRoot'],
+ # # 'WorkspacePathout': sys_config['WorkspacePathout'],
+ # # 'MetoFTP': sys_config['MetoFTP'],
+ # # 'WorkspacePath': sys_config['WorkspacePath'],
+ # # 'ResourcesPath': sys_config['ResourcesPath']}
+ #
+ # return sys_config
def prepare_job_directory(self, job_path):
"""
@@ -221,6 +215,7 @@ class ProcessorBase:
# check initial state of each config file, and gather terms that must apply
# across all provided configs
+ sys_config_path: str = args['sys_config_path']
config_paths: List[str] = args['config_paths']
component: str = args['component']
short_name: str = args['short_name']
@@ -228,12 +223,12 @@ class ProcessorBase:
clearup: bool = args['clearup']
# load universal configuration
- universal_config = self.build_universal_config(config_paths, component)
- universal_config['StartString'] = start_date
+ sys_config = parse_json_file_with_tokens(sys_config_path)
+ sys_config['StartString'] = start_date
# determine job directory
- workspacePath = universal_config['WorkspacePathout']
- job_path: str = f'{workspacePath}{short_name}_{start_date}'
+ workspace_path = sys_config['WorkspacePathout']
+ job_path: str = f'{workspace_path}{short_name}_{start_date}'
self.prepare_job_directory(job_path)
is_live: bool = args["live"]
@@ -246,9 +241,9 @@ class ProcessorBase:
logger.info("==========")
logger.info(f"Logging started at {datetime.datetime.now().strftime('%Y %b %d %H:%M:%S')}")
- logger.info(f"Universal config is\n{json.dumps(universal_config, indent = 2)}")
+ logger.info(f"Universal config is\n{json.dumps(sys_config, indent = 2)}")
logger.info(f"Job path will be {job_path}")
- workspacePath = universal_config['WorkspacePathout']
+ workspace_path = sys_config['WorkspacePathout']
# note that file date represents preceding 3 hours, so day's data starts at file timestamp 0300 UTC
start_time: datetime = datetime.datetime.strptime(start_date + '03', '%Y%m%d%H')
@@ -290,7 +285,14 @@ class ProcessorBase:
logger.info(f'Working on config {configIndex + 1} of {config_paths_length}')
try:
- configjson: dict = open_and_check_config(configtemplate)
+ configjson: dict = parse_json_file_with_tokens(configtemplate, sys_config)
+ # then add the sys_config keys and values to the configjson
+ for k, v in sys_config.items():
+ if k not in configjson.keys():
+ configjson[k] = v
+ else:
+ raise Exception(f"Overwriting config key {k} with value from sys_config")
+
except:
logger.exception(f"Failure in opening or checking config {configtemplate}")
# TODO: This case should test flagdir.jobStatus.__exit__()
@@ -303,12 +305,12 @@ class ProcessorBase:
# from configtemplate create configFileName to describe the specific job
component: str = component
- configFileName = f"{os.path.basename(configtemplate).replace('.json', '')}_{component}"
+ config_file_name = f"{os.path.basename(configtemplate).replace('.json', '')}_{component}"
- configjson['ConfigFilePath'] = configFileName
+ configjson['ConfigFilePath'] = config_file_name
# write the complete configuration file to job directory
- with open(f"{job_path}/{configFileName}.json", 'w') as write_file:
+ with open(f"{job_path}/{config_file_name}.json", 'w') as write_file:
json.dump(configjson, write_file, indent = 4)
# proc_description = universal_config['ProcessInJob']
@@ -353,7 +355,7 @@ class ProcessorBase:
if status.is_success() & (clearup is True):
logger.info('Clearing up')
- clearup_dest_dir = f"{workspacePath}/clearup/{short_name}_{start_date}/"
+ clearup_dest_dir = f"{workspace_path}/clearup/{short_name}_{start_date}/"
Path(clearup_dest_dir).mkdir(parents = True, exist_ok = True)
logger.info(f"While developing, moving directories to this directory : {clearup_dest_dir}")
diff --git a/ews/coordinator/processor_deposition.py b/ews/coordinator/processor_deposition.py
index 1d0be30..93e30c7 100644
--- a/ews/coordinator/processor_deposition.py
+++ b/ews/coordinator/processor_deposition.py
@@ -44,8 +44,8 @@ class ProcessorDeposition(ProcessorBase):
def process_in_job_dep(self, jobPath, status, config, component):
logger.info('started process_in_job_dep()')
- file_path = Template(config[component]['ServerPathTemplate']).substitute(**config)
- file_name = Template(config[component]['InputFileTemplate']).substitute(**config)
+ file_path = Template(config['ServerPathTemplate']).substitute(**config)
+ file_name = Template(config['InputFileTemplate']).substitute(**config)
logger.info(f"Expecting to work with {file_name}")
@@ -97,7 +97,7 @@ class ProcessorDeposition(ProcessorBase):
return proc_out
- def process_EWS_plotting_dep(self, jobPath, config) -> [str]:
+ def process_EWS_plotting_dep(self, job_path, config) -> [str]:
'''Returns a list of output files for transfer.'''
logger.info('started process_EWS_plotting_dep()')
@@ -105,29 +105,29 @@ class ProcessorDeposition(ProcessorBase):
# initialise environment
regions = config['SubRegionNames']
- deposition_file_name = Template(config['Deposition']['InputFileTemplate']).substitute(**config)
+ deposition_file_name = Template(config['InputFileTemplate']).substitute(**config)
- deposition_path = f"{jobPath}/{deposition_file_name}"
+ deposition_path = f"{job_path}/{deposition_file_name}"
# get the file name from the config
# this file name can be a glob, as long as matches can all be loaded by iris
- deposition_data_file_name = Template(config['Deposition']['DataFileTemplate']).substitute(**config)
+ deposition_data_file_name = Template(config['DataFileTemplate']).substitute(**config)
name_file_wildcard = f"{deposition_path}/{deposition_data_file_name}"
ews_plotting_output_globs = []
for region in regions:
- output_dir = f"{jobPath}/plotting/{region.lower()}"
+ output_dir = f"{job_path}/plotting/{region.lower()}"
Path(output_dir).mkdir(parents=True, exist_ok=True)
- sys_config = config['Deposition']['EWS-Plotting']['SysConfig']
- name_extraction_config = config['Deposition']['EWS-Plotting']['NameExtractionConfig']
- run_config = config['Deposition']['EWS-Plotting']['RunConfig']
- run_config_norm = config['Deposition']['EWS-Plotting']['RunConfigNorm']
- chart_config = config['Deposition']['EWS-Plotting'][region]['ChartConfig']
- normalize = config['Deposition']['EWS-Plotting'][region]['Normalize']
+ sys_config = config['EWS-Plotting']['SysConfig']
+ name_extraction_config = config['EWS-Plotting']['NameExtractionConfig']
+ run_config = config['EWS-Plotting']['RunConfig']
+ run_config_norm = config['EWS-Plotting']['RunConfigNorm']
+ chart_config = config['EWS-Plotting'][region]['ChartConfig']
+ normalize = config['EWS-Plotting'][region]['Normalize']
extraction_file_prefix = 'deposition_' + region.lower()
# Note that this runs all disease types available
diff --git a/ews/coordinator/processor_environment.py b/ews/coordinator/processor_environment.py
index de5d90b..99f93bd 100644
--- a/ews/coordinator/processor_environment.py
+++ b/ews/coordinator/processor_environment.py
@@ -60,10 +60,10 @@ class ProcessorEnvironment(ProcessorBase):
region = config['RegionName']
- pipeline_config = config["Environment"]
+ # pipeline_config = config["Environment"]
try:
logger.info(f"Calling environmental suitability 2.0 for {region}")
- self.run_pipeline(pipeline_config, region, config["StartString"])
+ self.run_pipeline(config, region, config["StartString"])
except:
logger.exception(f"Some failure when running EnvSuitPipeline.py")
raise
@@ -316,10 +316,10 @@ class ProcessorEnvironment(ProcessorBase):
Path(output_dir).mkdir(parents=True, exist_ok=True)
- sys_config = config['Environment']['EWS-Plotting']['SysConfig']
- run_config = config['Environment']['EWS-Plotting']['RunConfig']
- chart_config = config['Environment']['EWS-Plotting'][region]['ChartConfig']
- filter_for_country = config['Environment']['EWS-Plotting'][region]['FilterForCountry']
+ sys_config = config['EWS-Plotting']['SysConfig']
+ run_config = config['EWS-Plotting']['RunConfig']
+ chart_config = config['EWS-Plotting'][region]['ChartConfig']
+ filter_for_country = config['EWS-Plotting'][region]['FilterForCountry']
# Note that this runs all disease types available
diff --git a/ews/coordinator/processor_epidemiology.py b/ews/coordinator/processor_epidemiology.py
index 9dccc33..b9380fd 100644
--- a/ews/coordinator/processor_epidemiology.py
+++ b/ews/coordinator/processor_epidemiology.py
@@ -27,10 +27,9 @@ from ews.postprocessing.epi.epi_post_processor import EPIPostPostProcessor
from ews.coordinator.utils.processor_utils import (
calc_epi_date_range,
- open_and_check_config,
get_only_existing_globs,
end_job,
- disease_latin_name_dict, short_name
+ disease_latin_name_dict, short_name, parse_json_file_with_tokens
)
logger = logging.getLogger(__name__)
@@ -66,11 +65,11 @@ class ProcessorEpidemiology(ProcessorBase):
for configFile in config_fns:
# they should be working if the script made it this far, no need to try
- config_i = open_and_check_config(configFile)
+ config_i = parse_json_file_with_tokens(configFile)
#determine end time, from config file
arg_start_date: str = input_args['start_date']
- calc_span_days = config_i['Epidemiology']['CalculationSpanDays']
+ calc_span_days = config_i['CalculationSpanDays']
assert len(calc_span_days) == 2
start_time, end_time = calc_epi_date_range(arg_start_date,calc_span_days)
@@ -90,7 +89,7 @@ class ProcessorEpidemiology(ProcessorBase):
# create a string describing every epi calc configuration
epiStrings = []
- for epiconf in config['Epidemiology']['Epi']:
+ for epiconf in config['Epi']:
epiKwargsString = ''.join([f"{k}{v}" for k,v in epiconf['modelArguments'].items()])
# drop any repetitive elements of kwarg
@@ -282,22 +281,22 @@ class ProcessorEpidemiology(ProcessorBase):
# start date to reference date defines the range of the job that is being
# continued.
- # - Each model named under config['Epidemiology']['Epi'] will need a
+ # - Each model named under config['Epi'] will need a
# corresponding set of arguments for reading in as for depo and env, e.g.
- # config['Epidemiology']['model_name'].
+ # config['model_name'].
# - depo and env file listers will be applied only for the reference date
# to end date, so choose an apporporiate lister, e.g.
# list_onefile_operational
- is_continue = config['Epidemiology'].get('continue',False)
+ is_continue = config.get('continue',False)
# initialise any needed variables
reference_date_str = config['StartString']
reference_date = datetime.datetime.strptime(reference_date_str,'%Y%m%d')
- start_date, end_date = calc_epi_date_range(reference_date_str,config['Epidemiology']['CalculationSpanDays'])
+ start_date, end_date = calc_epi_date_range(reference_date_str,config['CalculationSpanDays'])
start_string = start_date.strftime('%Y-%m-%d-%H%M')
start_string_short = start_date.strftime('%Y%m%d%H%M')
@@ -313,10 +312,10 @@ class ProcessorEpidemiology(ProcessorBase):
yesterday_date = datetime.datetime.strptime(reference_date_str,'%Y%m%d') - datetime.timedelta(days=1)
yesterday_string = yesterday_date.strftime('%Y%m%d')
- diseases = config['Epidemiology']['DiseaseNames']
+ diseases = config['DiseaseNames']
# get list of variable names to be loaded from deposition input
- depo_variable_names = config['Epidemiology']['Deposition']['VariableNames']
+ depo_variable_names = config['Deposition']['VariableNames']
assert len(depo_variable_names) == len(diseases)
# loop over each sub region
@@ -331,7 +330,7 @@ class ProcessorEpidemiology(ProcessorBase):
config['SubRegionName'] = region
config['DiseaseName'] = disease
- config_epi = config['Epidemiology'].copy()
+ config_epi = config.copy()
# TODO: CAUTION: Any iterations (e.g. disease or sub-region) are hidden
# in jobPath, and not retained in the config file. This is a provlem for
@@ -665,31 +664,31 @@ class ProcessorEpidemiology(ProcessorBase):
# initalise necessary variables from config
- start_date, end_date = calc_epi_date_range(config['StartString'],config['Epidemiology']['CalculationSpanDays'])
+ start_date, end_date = calc_epi_date_range(config['StartString'],config['CalculationSpanDays'])
start_string = start_date.strftime('%Y%m%d')
end_string = end_date.strftime('%Y%m%d')
- epi_case_operational = config['Epidemiology']['EWS-Plotting']['EpiCase']
+ epi_case_operational = config['EWS-Plotting']['EpiCase']
if epi_case_operational == 'none':
logger.info('Config specifies not to call to EWS-Plotting')
return []
- diseases = config['Epidemiology']['DiseaseNames']
+ diseases = config['DiseaseNames']
# initialise environment
- sys_config = config['Epidemiology']['EWS-Plotting']['SysConfig']
+ sys_config = config['EWS-Plotting']['SysConfig']
- chart_config = config['Epidemiology']['EWS-Plotting']['ChartConfig']
+ chart_config = config['EWS-Plotting']['ChartConfig']
# use the first matching epi formulation
# TODO: Is there a more efficient way to select?
- epi_filename = [ce['infectionRasterFileName'] for ce in config['Epidemiology']['Epi'] if ce['model']==epi_case_operational][0]
+ epi_filename = [ce['infectionRasterFileName'] for ce in config['Epi'] if ce['model']==epi_case_operational][0]
dep_regionnames = ['SouthAsia','Ethiopia']
- # TODO get deposition_dir from config['Epidemiology']['Deposition']['PathTemplate']
+ # TODO get deposition_dir from config['Deposition']['PathTemplate']
dep_regionname = 'Ethiopia' #SouthAsia
deposition_dir = f"{config['WorkspacePath']}DEPOSITION_{start_string}/WR_NAME_{dep_regionname}_{start_string}/"
@@ -707,7 +706,7 @@ class ProcessorEpidemiology(ProcessorBase):
disease_short = disease.lower().replace('rust','')
# a fudge, guess disease type
- # because config['Epidemiology']['ProcessInJob'] handles disease loop internally
+ # because config['ProcessInJob'] handles disease loop internally
# assumes disease name is the last directory before the filename
# TODO: handle multiple diseases and regions in Processor as a loop, or in the config
disease_to_drop = os.path.dirname(epi_filename).split('/')[-1].replace('Rust','')
@@ -715,16 +714,16 @@ class ProcessorEpidemiology(ProcessorBase):
epi_filename = epi_filename.replace(disease_to_drop,disease_to_add)
map_title = "Integrated prediction of Wheat $\\bf{" + disease_to_add + "}$ Rust infection"
- if 'PlottingRegionName' not in config['Epidemiology']['EWS-Plotting']:
+ if 'PlottingRegionName' not in config['EWS-Plotting']:
plotting_region_name_lower = config['RegionName'].lower()
else:
- plotting_region_name_lower = config['Epidemiology']['EWS-Plotting']['PlottingRegionName'].lower()
+ plotting_region_name_lower = config['EWS-Plotting']['PlottingRegionName'].lower()
epi_seasonsofar_fn = epi_filename+'_per_ha_wheat_seasonsofar.csv'
epi_seasonincforecast_fn = epi_filename+'_per_ha_wheat.csv'
- seasonsofar_run_config = config['Epidemiology']['EWS-Plotting'].get('RunConfig_seasonsofar',None)
+ seasonsofar_run_config = config['EWS-Plotting'].get('RunConfig_seasonsofar',None)
# only run season so far (i.e. historic dates) if they exist
if (seasonsofar_run_config is not None) & os.path.exists(epi_seasonsofar_fn):
@@ -747,7 +746,7 @@ class ProcessorEpidemiology(ProcessorBase):
# prepare command for seasonplusforecast
- run_config = config['Epidemiology']['EWS-Plotting']['RunConfig_seasonplusforecast']
+ run_config = config['EWS-Plotting']['RunConfig_seasonplusforecast']
logger.info(f"Running EWS-Plotting with the following configs:\n{sys_config}\n{run_config}\n{chart_config}")
diff --git a/ews/coordinator/processor_surveys.py b/ews/coordinator/processor_surveys.py
index dbc667a..8bc4274 100644
--- a/ews/coordinator/processor_surveys.py
+++ b/ews/coordinator/processor_surveys.py
@@ -83,7 +83,7 @@ class ProcessorSurveys(ProcessorBase):
logger.debug('Performing download(s) from ODK server')
- credentials_filename = config['Survey']['ServerCredentialsFile']
+ credentials_filename = config['ServerCredentialsFile']
with open(credentials_filename) as credentials_file:
cred: dict = json.load(credentials_file)
@@ -148,9 +148,9 @@ class ProcessorSurveys(ProcessorBase):
# simple format alignment using edits on config
# (should this need to be much more sophisticated, reconsider the workflow)
- if 'FormEdits' in config['Survey']:
+ if 'FormEdits' in config:
- form_edits = config['Survey']['FormEdits']
+ form_edits = config['FormEdits']
# loop over each form
for form_name, edits in form_edits.items():
@@ -264,7 +264,7 @@ class ProcessorSurveys(ProcessorBase):
upload_directory = f"{jobPath}/upload"
Path(upload_directory).mkdir(parents=True, exist_ok=True)
- if 'Groups' in config['Survey']:
+ if 'Groups' in config:
# if 'Groups' is defined in the config, create grouped survey files and run python version
logger.info('Preparing grouped survey files')
@@ -272,12 +272,12 @@ class ProcessorSurveys(ProcessorBase):
Path(group_directory).mkdir(parents=True, exist_ok=True)
# creating initial groups
- groups = config['Survey']['Groups']
+ groups = config['Groups']
# check if columns requested in GroupBy are present in the dataframe
- assert all([group_by in df_join.columns for group_by in config['Survey']['GroupBy']]), 'Column(s) requested in GroupBy are not present in the dataframe'
+ assert all([group_by in df_join.columns for group_by in config['GroupBy']]), 'Column(s) requested in GroupBy are not present in the dataframe'
- for group_by in config['Survey']['GroupBy']:
+ for group_by in config['GroupBy']:
logger.debug(f"grouping by {group_by}")
# handle NaN values
@@ -294,8 +294,8 @@ class ProcessorSurveys(ProcessorBase):
groups.update({i:{group_by:[i]} for i in groups_list})
# remove groups that are listed in GroupsToIgnore
- if 'GroupsToIgnore' in config['Survey']:
- groups_to_ignore = config['Survey']['GroupsToIgnore']
+ if 'GroupsToIgnore' in config:
+ groups_to_ignore = config['GroupsToIgnore']
# add groups to ignore if all elements of the group are in the list of groups to ignore
for group_name in groups:
@@ -327,18 +327,18 @@ class ProcessorSurveys(ProcessorBase):
output_directory = f"{jobPath}/source_gen/{group_name}"
Path(output_directory).mkdir(parents=True, exist_ok=True)
- if 'SourcesConfigs' in config['Survey'] and group_name in config['Survey']['SourcesConfigs']:
- logger.info(f"Running source gen for {group_name} group wih config {config['Survey']['SourcesConfigs'][group_name]}")
- sources_config = config['Survey']['SourcesConfigs'][group_name]
+ if 'SourcesConfigs' in config and group_name in config['SourcesConfigs']:
+ logger.info(f"Running source gen for {group_name} group wih config {config['SourcesConfigs'][group_name]}")
+ sources_config = config['SourcesConfigs'][group_name]
else:
- logger.info(f"Running source gen for {group_name} group wih default config {config['Survey']['SourcesConfigDefault']}")
- sources_config = config['Survey']['SourcesConfigDefault']
+ logger.info(f"Running source gen for {group_name} group wih default config {config['SourcesConfigDefault']}")
+ sources_config = config['SourcesConfigDefault']
sources_path = run_case(
config_path = sources_config,
survey_path = group_surveys_filepath,
- survey_format = config['Survey']['SurveyFormat'],
- surveyor_name_col = config['Survey']['SurveyorNameCol'],
+ survey_format = config['SurveyFormat'],
+ surveyor_name_col = config['SurveyorNameCol'],
reference_date = config['StartString'],
# Day 0 (current day) is always included
# Days -2,-1 and 0 are needed to initialise spores in NAME
@@ -374,10 +374,10 @@ class ProcessorSurveys(ProcessorBase):
Path(output_directory).mkdir(parents=True, exist_ok=True)
sources_path = run_case(
- config_path = config['Survey']['SourcesConfigDefault'],
+ config_path = config['SourcesConfigDefault'],
survey_path = processed_surveys_filepath,
- survey_format = config['Survey']['SurveyFormat'],
- surveyor_name_col = config['Survey']['SurveyorNameCol'],
+ survey_format = config['SurveyFormat'],
+ surveyor_name_col = config['SurveyorNameCol'],
reference_date = config['StartString'],
# Day 0 (current day) is always included
# Days -2,-1 and 0 are needed to initialise spores in NAME
diff --git a/ews/coordinator/survey_servers/processor_surveys_kobotoolbox.py b/ews/coordinator/survey_servers/processor_surveys_kobotoolbox.py
index b9f96c5..2fb8db6 100644
--- a/ews/coordinator/survey_servers/processor_surveys_kobotoolbox.py
+++ b/ews/coordinator/survey_servers/processor_surveys_kobotoolbox.py
@@ -145,7 +145,7 @@ def get_kobotoolbox_form_as_csv(form_credentials: dict, jobPath: str, config: di
download_success = True
- skip_download: bool = config['Survey'].get('SkipServerDownload', False)
+ skip_download: bool = config.get('SkipServerDownload', False)
if not skip_download:
try:
@@ -189,7 +189,7 @@ def get_kobotoolbox_form_as_csv(form_credentials: dict, jobPath: str, config: di
copy_success = False
days_back = 1
- acceptable_days_back = int(config['Survey']['AcceptableDowntimeDays'])
+ acceptable_days_back = int(config['AcceptableDowntimeDays'])
logger.debug(f"Acceptable server downtime is set to {acceptable_days_back} days")
while ((not copy_success) and (days_back <= acceptable_days_back)):
diff --git a/ews/coordinator/survey_servers/processor_surveys_new_odk.py b/ews/coordinator/survey_servers/processor_surveys_new_odk.py
index 5746843..22f2804 100644
--- a/ews/coordinator/survey_servers/processor_surveys_new_odk.py
+++ b/ews/coordinator/survey_servers/processor_surveys_new_odk.py
@@ -171,7 +171,7 @@ def get_newODK_form_as_csv(form_credentials: dict, jobPath: str, config: dict, s
download_success = True
- skip_download: bool = config['Survey'].get('SkipServerDownload', False)
+ skip_download: bool = config.get('SkipServerDownload', False)
if not skip_download:
try:
@@ -215,7 +215,7 @@ def get_newODK_form_as_csv(form_credentials: dict, jobPath: str, config: dict, s
copy_success = False
days_back = 1
- acceptable_days_back = int(config['Survey']['AcceptableDowntimeDays'])
+ acceptable_days_back = int(config['AcceptableDowntimeDays'])
logger.debug(f"Acceptable server downtime is set to {acceptable_days_back} days")
while ((not copy_success) and (days_back <= acceptable_days_back)):
diff --git a/ews/coordinator/survey_servers/processor_surveys_new_odk2.py b/ews/coordinator/survey_servers/processor_surveys_new_odk2.py
index c5b6ea4..3a0f40b 100644
--- a/ews/coordinator/survey_servers/processor_surveys_new_odk2.py
+++ b/ews/coordinator/survey_servers/processor_surveys_new_odk2.py
@@ -172,7 +172,7 @@ def get_newODK2_form_as_csv(form_credentials: dict, jobPath: str, config: dict,
download_success = True
- skip_download: bool = config['Survey'].get('SkipServerDownload', False)
+ skip_download: bool = config.get('SkipServerDownload', False)
if not skip_download:
try:
@@ -229,7 +229,7 @@ def get_newODK2_form_as_csv(form_credentials: dict, jobPath: str, config: dict,
copy_success = False
days_back = 1
- acceptable_days_back = int(config['Survey']['AcceptableDowntimeDays'])
+ acceptable_days_back = int(config['AcceptableDowntimeDays'])
logger.debug(f"Acceptable server downtime is set to {acceptable_days_back} days")
while ((not copy_success) and (days_back <= acceptable_days_back)):
diff --git a/ews/coordinator/survey_servers/processor_surveys_odk.py b/ews/coordinator/survey_servers/processor_surveys_odk.py
index fcfe93e..c396872 100644
--- a/ews/coordinator/survey_servers/processor_surveys_odk.py
+++ b/ews/coordinator/survey_servers/processor_surveys_odk.py
@@ -21,7 +21,7 @@ def get_ODK_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat
'''Given a dict with a single ODK form to download from an ODK Aggregate
server, obtains it and converts to csv.'''
- ODK_output_path_template = config['Survey'].get('ODKDatabasePathTemplate','${WorkspacePathout}/ODK_DB/')
+ ODK_output_path_template = config.get('ODKDatabasePathTemplate','${WorkspacePathout}/ODK_DB/')
ODK_output_path = Template(ODK_output_path_template).substitute(**config)
# get data from ODK server
@@ -32,7 +32,7 @@ def get_ODK_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat
ODK_jar = form_credentials['ODK_jar']
assert os.path.exists(ODK_jar)
- skip_download: bool = config['Survey'].get('SkipServerDownload', False)
+ skip_download: bool = config.get('SkipServerDownload', False)
ODK_download_success = True
@@ -95,7 +95,7 @@ def get_ODK_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat
ODK_copy_success = False
days_back = 1
- acceptable_days_back = int(config['Survey']['AcceptableDowntimeDays'])
+ acceptable_days_back = int(config['AcceptableDowntimeDays'])
logger.debug(f"Acceptable server downtime is set to {acceptable_days_back} days")
while ((not ODK_copy_success) and (days_back <= acceptable_days_back)):
diff --git a/ews/coordinator/survey_servers/processor_surveys_odk_sa.py b/ews/coordinator/survey_servers/processor_surveys_odk_sa.py
index 1e035af..1b93de1 100644
--- a/ews/coordinator/survey_servers/processor_surveys_odk_sa.py
+++ b/ews/coordinator/survey_servers/processor_surveys_odk_sa.py
@@ -101,7 +101,7 @@ def get_ODK_SA_form_as_csv(form_credentials: dict, jobPath: str, config: dict, s
'''Given a dict with a single ODK form to download from an ODK Aggregate
server with South Asia formatting, obtains it and converts to csv.'''
- ODK_output_path_template = config['Survey'].get('ODKDatabasePathTemplate','${WorkspacePathout}/ODK_DB/')
+ ODK_output_path_template = config.get('ODKDatabasePathTemplate','${WorkspacePathout}/ODK_DB/')
ODK_output_path = Template(ODK_output_path_template).substitute(**config)
# get data from ODK server
@@ -112,7 +112,7 @@ def get_ODK_SA_form_as_csv(form_credentials: dict, jobPath: str, config: dict, s
ODK_jar = form_credentials['ODK_jar']
assert os.path.exists(ODK_jar)
- skip_download: bool = config['Survey'].get('SkipServerDownload', False)
+ skip_download: bool = config.get('SkipServerDownload', False)
ODK_download_success = True
@@ -207,7 +207,7 @@ def get_ODK_SA_form_as_csv(form_credentials: dict, jobPath: str, config: dict, s
ODK_copy_success = False
days_back = 1
- acceptable_days_back = int(config['Survey']['AcceptableDowntimeDays'])
+ acceptable_days_back = int(config['AcceptableDowntimeDays'])
logger.debug(f"Acceptable server downtime is set to {acceptable_days_back} days")
while ((not ODK_copy_success) and (days_back <= acceptable_days_back)):
diff --git a/ews/coordinator/survey_servers/processor_surveys_odk_v2.py b/ews/coordinator/survey_servers/processor_surveys_odk_v2.py
index 8c7fd3b..031eac4 100644
--- a/ews/coordinator/survey_servers/processor_surveys_odk_v2.py
+++ b/ews/coordinator/survey_servers/processor_surveys_odk_v2.py
@@ -335,7 +335,7 @@ def get_ODKv2_form_as_csv(form_credentials: dict, jobPath: str, config: dict, st
download_success = True
- skip_download: bool = config['Survey'].get('SkipServerDownload', False)
+ skip_download: bool = config.get('SkipServerDownload', False)
if not skip_download:
try:
@@ -386,7 +386,7 @@ def get_ODKv2_form_as_csv(form_credentials: dict, jobPath: str, config: dict, st
copy_success = False
days_back = 1
- acceptable_days_back = int(config['Survey']['AcceptableDowntimeDays'])
+ acceptable_days_back = int(config['AcceptableDowntimeDays'])
logger.debug(f"Acceptable server downtime is set to {acceptable_days_back} days")
while ((not copy_success) and (days_back <= acceptable_days_back)):
diff --git a/ews/coordinator/survey_servers/processor_surveys_wrsis.py b/ews/coordinator/survey_servers/processor_surveys_wrsis.py
index 9411ac2..7d335d3 100644
--- a/ews/coordinator/survey_servers/processor_surveys_wrsis.py
+++ b/ews/coordinator/survey_servers/processor_surveys_wrsis.py
@@ -263,12 +263,12 @@ def get_WRSIS_form_as_csv(form_credentials: dict, jobPath: str, config: dict, st
download_success = True
- start_date = datetime.datetime.strptime(config['Survey']['SeasonStartString'],'%Y%m%d').strftime('%d-%m-%Y')
+ start_date = datetime.datetime.strptime(config['SeasonStartString'],'%Y%m%d').strftime('%d-%m-%Y')
end_date = datetime.datetime.strptime(config['StartString'], '%Y%m%d').strftime('%d-%m-%Y')
logger.debug(f'Performing download from WRSIS between {start_date} and {end_date}')
- skip_download: bool = config['Survey'].get('SkipServerDownload', False)
+ skip_download: bool = config.get('SkipServerDownload', False)
if not skip_download:
try:
@@ -319,7 +319,7 @@ def get_WRSIS_form_as_csv(form_credentials: dict, jobPath: str, config: dict, st
copy_success = False
days_back = 1
- acceptable_days_back = int(config['Survey']['AcceptableDowntimeDays'])
+ acceptable_days_back = int(config['AcceptableDowntimeDays'])
logger.debug(f"Acceptable server downtime is set to {acceptable_days_back} days")
while ((not copy_success) and (days_back <= acceptable_days_back)):
diff --git a/ews/coordinator/survey_servers/processor_surveys_wrt.py b/ews/coordinator/survey_servers/processor_surveys_wrt.py
index 13f7f3e..8130569 100644
--- a/ews/coordinator/survey_servers/processor_surveys_wrt.py
+++ b/ews/coordinator/survey_servers/processor_surveys_wrt.py
@@ -240,12 +240,12 @@ def get_WRT_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat
download_success = True
- start_date = datetime.datetime.strptime(config['Survey']['SeasonStartString'],'%Y%m%d').strftime('%Y-%m-%d')
+ start_date = datetime.datetime.strptime(config['SeasonStartString'],'%Y%m%d').strftime('%Y-%m-%d')
end_date = datetime.datetime.strptime(config['StartString'], '%Y%m%d').strftime('%Y-%m-%d')
logger.debug(f'Performing download from WRT between {start_date} and {end_date}')
- skip_download: bool = config['Survey'].get('SkipServerDownload', False)
+ skip_download: bool = config.get('SkipServerDownload', False)
if not skip_download:
try:
@@ -300,7 +300,7 @@ def get_WRT_form_as_csv(form_credentials: dict, jobPath: str, config: dict, stat
copy_success = False
days_back = 1
- acceptable_days_back = int(config['Survey']['AcceptableDowntimeDays'])
+ acceptable_days_back = int(config['AcceptableDowntimeDays'])
logger.debug(f"Acceptable server downtime is set to {acceptable_days_back} days")
while ((not copy_success) and (days_back <= acceptable_days_back)):
diff --git a/ews/coordinator/utils/processor_utils.py b/ews/coordinator/utils/processor_utils.py
index 235843c..5f488d5 100644
--- a/ews/coordinator/utils/processor_utils.py
+++ b/ews/coordinator/utils/processor_utils.py
@@ -125,6 +125,22 @@ def open_and_check_config(configFile) -> dict:
return config
+def parse_json_file_with_tokens(file_path: str, tokens: dict = None) -> dict:
+
+ with open(file_path, 'r') as file:
+ file_string: str = file.read()
+ file.seek(0) # reset the index to read again
+ file_json_dict = json.load(file)
+ # use the file string as a template and replace any tokens (marked as ${string}) with any keys from the
+ # file_json_dict that match the tokens
+ parsed_file_string: str = Template(file_string).safe_substitute(**file_json_dict)
+ if tokens is not None:
+ # reparse the string with any other tokens passed in as arguments
+ parsed_file_string: str = Template(parsed_file_string).safe_substitute(**tokens)
+ # finally, parse the tokenized string as a json dict
+ result = json.loads(parsed_file_string)
+
+ return result
def get_only_existing_globs(file_globs: [str], inplace = True):
@@ -411,7 +427,6 @@ def get_input_data(job_path, config, component):
copies input files to job dir, using file name templates defined in the config file
:param job_path:
:param config:
- :param component:
:return:
"""
@@ -424,8 +439,8 @@ def get_input_data(job_path, config, component):
# Processing files available for clearing
proc_out['clearup'] = []
- file_path = Template(config[component]['ServerPathTemplate']).substitute(**config)
- file_name = Template(config[component]['InputFileTemplate']).substitute(**config)
+ file_path = Template(config['ServerPathTemplate']).substitute(**config)
+ file_name = Template(config['InputFileTemplate']).substitute(**config)
#TODO: check if file exists already (may be the case for multiple configs in one)
diff --git a/tests/integration/full/full_test_advisory.py b/tests/integration/full/full_test_advisory.py
index f8f76c4..915c146 100644
--- a/tests/integration/full/full_test_advisory.py
+++ b/tests/integration/full/full_test_advisory.py
@@ -58,8 +58,8 @@ class FullTestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
# may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
+ IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME)
@staticmethod
diff --git a/tests/integration/full/full_test_deposition.py b/tests/integration/full/full_test_deposition.py
index d1ad040..6c3e0bd 100644
--- a/tests/integration/full/full_test_deposition.py
+++ b/tests/integration/full/full_test_deposition.py
@@ -3,6 +3,7 @@ import os
import sys
from ews.coordinator.processor_deposition import ProcessorDeposition
+from ews.coordinator.utils.processor_utils import parse_json_file_with_tokens
from integration.partial.integration_test_utils import IntegrationTestUtils
from integration.test_suites.depo_test_suite import BaseDepoTestSuite
@@ -32,16 +33,22 @@ class FullTestDeposition(BaseDepoTestSuite.DepoTestSuite):
@staticmethod
def write_temp_run_config_file():
- default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
- default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
- run_dict: dict = copy.deepcopy(default_config_dict)
- # BaseDepoTestSuite.DepoTestSuite.TEST_OUT_PATH = IntegrationTestUtils.TEST_WORKSPACE_PATH + prefix + os.sep
+ sys_config = IntegrationTestUtils.DEFAULT_SYS_CONFIG_FILE_PATH
+ sys_config_dict: dict = parse_json_file_with_tokens(sys_config)
+
+ os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
+ IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+ IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME)
+
+ run_config = IntegrationTestUtils.DEFAULT_DEPO_CONFIG_FILE_PATH
+ run_config_temp_dict: dict = parse_json_file_with_tokens(run_config)
+ run_dict: dict = copy.deepcopy(run_config_temp_dict)
+
run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
- os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
+ IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME)
@staticmethod
diff --git a/tests/integration/full/full_test_env_suit.py b/tests/integration/full/full_test_env_suit.py
index fc8d806..9537da4 100644
--- a/tests/integration/full/full_test_env_suit.py
+++ b/tests/integration/full/full_test_env_suit.py
@@ -41,8 +41,8 @@ class FullTestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
run_dict['Environment']['EWS-Plotting']['Kenya']['FilterForCountry'] = "True"
os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
+ IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME)
@staticmethod
diff --git a/tests/integration/full/full_test_epi.py b/tests/integration/full/full_test_epi.py
index ca2fa3c..9acfd8e 100644
--- a/tests/integration/full/full_test_epi.py
+++ b/tests/integration/full/full_test_epi.py
@@ -64,8 +64,8 @@ class FullTestEpi(BaseEpiTestSuite.EpiTestSuite):
# may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
+ IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME)
@staticmethod
diff --git a/tests/integration/full/full_test_survey.py b/tests/integration/full/full_test_survey.py
index 95f81d0..9cd082e 100644
--- a/tests/integration/full/full_test_survey.py
+++ b/tests/integration/full/full_test_survey.py
@@ -36,8 +36,8 @@ class FullTestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
run_dict['Survey']['SkipServerDownload'] = False
os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
+ IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME)
@staticmethod
def run_survey_pipeline():
diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py
index 354e1e8..e2e0989 100644
--- a/tests/integration/partial/integration_test_utils.py
+++ b/tests/integration/partial/integration_test_utils.py
@@ -17,9 +17,16 @@ class IntegrationTestUtils:
EMAIL_CRED_PATH: str = "../../test_data/test_deployment/envs/Cred_gmail.json"
LOGGING_CONFIG_PATH: str = "../../test_data/test_deployment/envs/test_log_config.json"
- DEFAULT_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json"
+ DEFAULT_SYS_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/sys_config_EastAfrica_fc_live.json"
+ DEFAULT_DEPO_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/depo_config_EastAfrica_fc_live.json"
+ DEFAULT_ENV_SUIT_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/env_suit_config_EastAfrica_fc_live.json"
+ DEFAULT_EPI_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/epi_config_EastAfrica_fc_live.json"
+ DEFAULT_SURVEY_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/survey_config_EastAfrica_fc_live.json"
+ DEFAULT_ADVISORY_CONFIG_FILE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/advisory_config_EastAfrica_fc_live.json"
+
TEST_WORKSPACE_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/workspace/"
- TEMP_CONFIG_FILE_NAME: str = None
+ TEMP_SYS_CONFIG_FILE_NAME: str = None
+ TEMP_RUN_CONFIG_FILE_NAME: str = None
TEST_ASSETS_PATH: str = "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/assets/"
EXAMPLE_SURVEY_FILE_PATH: str = TEST_ASSETS_PATH + "example_survey_run.zip"
@@ -190,7 +197,8 @@ class IntegrationTestUtils:
args_dict: dict = {}
- config_paths = [IntegrationTestUtils.TEMP_CONFIG_FILE_NAME]
+ sys_config_path = IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME
+ config_paths = [IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME]
# note, possible to override these values in the kwargs loop below
args_dict['live'] = False
@@ -198,6 +206,7 @@ class IntegrationTestUtils:
args_dict['start_date'] = start_date
args_dict['component'] = component
args_dict['short_name'] = shortname
+ args_dict['sys_config_path'] = sys_config_path
args_dict['config_paths'] = config_paths
args_dict['log_level'] = 'info'
args_dict['clearup'] = True
@@ -231,7 +240,7 @@ class IntegrationTestUtils:
args_dict['start_date'] = start_date
args_dict['component'] = component
args_dict['short_name'] = short_name
- args_dict['config_paths'] = [IntegrationTestUtils.TEMP_CONFIG_FILE_NAME]
+ args_dict['config_paths'] = [IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME]
args_dict['log_level'] = 'info'
args_dict['clearup'] = True
diff --git a/tests/integration/partial/test_advisory.py b/tests/integration/partial/test_advisory.py
index d72450e..79c86e0 100644
--- a/tests/integration/partial/test_advisory.py
+++ b/tests/integration/partial/test_advisory.py
@@ -44,19 +44,24 @@ class TestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
prefix: str = "temp_advisory_" + nowstring
# prefix: str = "temp_advisory"
- default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
- default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
- run_dict: dict = copy.deepcopy(default_config_dict)
IntegrationTestUtils.TEST_OUT_PATH = IntegrationTestUtils.TEST_WORKSPACE_PATH + prefix + os.sep
- # TestAdvisory.TEST_OUT_PATH = run_dict['WorkspacePathout'] + prefix + os.sep
- run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['ServerName'] = '' # nothing, as local machine
+ os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
+
+ sys_config_path = IntegrationTestUtils.DEFAULT_SYS_CONFIG_FILE_PATH
+ sys_config_dict: dict = IntegrationTestUtils.load_json_file(sys_config_path)
+ sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
+ sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
+ IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+ IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME)
+
+ run_config = IntegrationTestUtils.DEFAULT_ADVISORY_CONFIG_FILE_PATH
+ run_config_dict: dict = IntegrationTestUtils.load_json_file(run_config)
+ IntegrationTestUtils.TEST_OUT_PATH = IntegrationTestUtils.TEST_WORKSPACE_PATH + prefix + os.sep
# may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
+ IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_config_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME)
@staticmethod
diff --git a/tests/integration/partial/test_deposition.py b/tests/integration/partial/test_deposition.py
index f4d9b20..ac21193 100644
--- a/tests/integration/partial/test_deposition.py
+++ b/tests/integration/partial/test_deposition.py
@@ -45,20 +45,25 @@ class TestDeposition(BaseDepoTestSuite.DepoTestSuite):
prefix: str = "temp_depo_" + nowstring
# prefix: str = "temp_depo"
- default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
- default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
- run_dict: dict = copy.deepcopy(default_config_dict)
+ # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
IntegrationTestUtils.TEST_OUT_PATH = IntegrationTestUtils.TEST_WORKSPACE_PATH + prefix + os.sep
- run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['ServerName'] = '' # nothing, as local machine
+ os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
+
+ sys_config_path = IntegrationTestUtils.DEFAULT_SYS_CONFIG_FILE_PATH
+ sys_config_dict: dict = IntegrationTestUtils.load_json_file(sys_config_path)
+ sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
+ sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
+ IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+ IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME)
+
+ run_config_path = IntegrationTestUtils.DEFAULT_DEPO_CONFIG_FILE_PATH
+ run_config_dict: dict = IntegrationTestUtils.load_json_file(run_config_path)
+ # run_config_dict: dict = copy.deepcopy(run_config_dict)
full_server_path = os.path.abspath(IntegrationTestUtils.TEST_ASSETS_PATH)
- run_dict['Deposition']['ServerPathTemplate'] = full_server_path
+ run_config_dict['ServerPathTemplate'] = full_server_path
- # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
- os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
+ IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_config_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME)
@staticmethod
diff --git a/tests/integration/partial/test_env_suit.py b/tests/integration/partial/test_env_suit.py
index 9ade40d..9153a62 100644
--- a/tests/integration/partial/test_env_suit.py
+++ b/tests/integration/partial/test_env_suit.py
@@ -45,23 +45,28 @@ class TestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
prefix: str = "temp_env_" + nowstring
# prefix: str = "temp_env"
- default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
- default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
- run_dict: dict = copy.deepcopy(default_config_dict)
+ # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
IntegrationTestUtils.TEST_OUT_PATH = IntegrationTestUtils.TEST_WORKSPACE_PATH + prefix + os.sep
- run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['Environment']['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['Environment']['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['Environment']['OUTPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['ServerName'] = '' # nothing, as local machine
+ os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
+
+ sys_config_path = IntegrationTestUtils.DEFAULT_SYS_CONFIG_FILE_PATH
+ sys_config_dict: dict = IntegrationTestUtils.load_json_file(sys_config_path)
+ sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
+ sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
+ IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+ IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME)
+
+ run_config_path = IntegrationTestUtils.DEFAULT_ENV_SUIT_CONFIG_FILE_PATH
+ run_config_dict: dict = IntegrationTestUtils.load_json_file(run_config_path)
+ # run_config_dict: dict = copy.deepcopy(run_config_dict)
+ run_config_dict['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
+ run_config_dict['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
+ run_config_dict['OUTPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
full_server_path = os.path.abspath(IntegrationTestUtils.TEST_ASSETS_PATH)
- run_dict['Environment']['ServerPathTemplate'] = full_server_path
+ run_config_dict['ServerPathTemplate'] = full_server_path
- # may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
- os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
+ IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_config_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME)
@staticmethod
diff --git a/tests/integration/partial/test_epi.py b/tests/integration/partial/test_epi.py
index b9466d6..8a61e8d 100644
--- a/tests/integration/partial/test_epi.py
+++ b/tests/integration/partial/test_epi.py
@@ -41,19 +41,23 @@ class TestEpi(BaseEpiTestSuite.EpiTestSuite):
prefix: str = "temp_epi_" + nowstring
# prefix: str = "temp_epi"
- default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
- default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
- run_dict: dict = copy.deepcopy(default_config_dict)
IntegrationTestUtils.TEST_OUT_PATH = IntegrationTestUtils.TEST_WORKSPACE_PATH + prefix + os.sep
- run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['ServerName'] = '' # nothing, as local machine
- run_dict['Epidemiology']['CalculationSpanDays'] = [0, 1]
-
os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- name = IntegrationTestUtils.TEMP_CONFIG_FILE_NAME
- IntegrationTestUtils.write_json_file(run_dict, name)
+
+ sys_config_path = IntegrationTestUtils.DEFAULT_SYS_CONFIG_FILE_PATH
+ sys_config_dict: dict = IntegrationTestUtils.load_json_file(sys_config_path)
+ sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
+ sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
+ IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+ IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME)
+
+ run_config = IntegrationTestUtils.DEFAULT_EPI_CONFIG_FILE_PATH
+ run_config_dict: dict = IntegrationTestUtils.load_json_file(run_config)
+ run_config_dict['ServerName'] = '' # nothing, as local machine
+ run_config_dict['CalculationSpanDays'] = [0, 1]
+
+ IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_config_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME)
@staticmethod
diff --git a/tests/integration/partial/test_survey.py b/tests/integration/partial/test_survey.py
index 52edb0f..59edcf4 100644
--- a/tests/integration/partial/test_survey.py
+++ b/tests/integration/partial/test_survey.py
@@ -42,21 +42,29 @@ class TestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
prefix: str = "temp_survey_" + nowstring
# prefix: str = "temp_survey"
- default_config = IntegrationTestUtils.DEFAULT_CONFIG_FILE_PATH
- default_config_dict: dict = IntegrationTestUtils.load_json_file(default_config)
- run_dict: dict = copy.deepcopy(default_config_dict)
IntegrationTestUtils.TEST_OUT_PATH = IntegrationTestUtils.TEST_WORKSPACE_PATH + prefix + os.sep
- run_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['Environment']['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['Environment']['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
- run_dict['Environment']['OUTPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
+ os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
+
+ sys_config_path = IntegrationTestUtils.DEFAULT_SYS_CONFIG_FILE_PATH
+ sys_config_dict: dict = IntegrationTestUtils.load_json_file(sys_config_path)
+ sys_config_dict['WorkspacePathout'] = IntegrationTestUtils.TEST_OUT_PATH
+ sys_config_dict['WorkspacePath'] = IntegrationTestUtils.TEST_OUT_PATH
+ IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_sys_config.json"
+ IntegrationTestUtils.write_json_file(sys_config_dict, IntegrationTestUtils.TEMP_SYS_CONFIG_FILE_NAME)
+
+ run_config = IntegrationTestUtils.DEFAULT_SURVEY_CONFIG_FILE_PATH
+ run_config_dict: dict = IntegrationTestUtils.load_json_file(run_config)
+ run_dict: dict = copy.deepcopy(run_config_dict)
+ IntegrationTestUtils.TEST_OUT_PATH = IntegrationTestUtils.TEST_WORKSPACE_PATH + prefix + os.sep
+ run_dict['WORK_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
+ run_dict['INPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
+ run_dict['OUTPUT_PATH'] = IntegrationTestUtils.TEST_OUT_PATH
run_dict['ServerName'] = '' # nothing, as local machine
# may be reusing a non-timestamped output file during development, so allow extant TEST_OUT_PATH
os.makedirs(IntegrationTestUtils.TEST_OUT_PATH, exist_ok = True)
- IntegrationTestUtils.TEMP_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
- IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_CONFIG_FILE_NAME)
+ IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME = IntegrationTestUtils.TEST_OUT_PATH + "temp_config.json"
+ IntegrationTestUtils.write_json_file(run_dict, IntegrationTestUtils.TEMP_RUN_CONFIG_FILE_NAME)
@staticmethod
def unpack_dependencies():
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/depo_config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/depo_config_EastAfrica_fc_live.json
new file mode 100644
index 0000000..2fe47aa
--- /dev/null
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/depo_config_EastAfrica_fc_live.json
@@ -0,0 +1,23 @@
+{
+ "ServerPathTemplate": "OVERRIDDEN",
+ "InputFileTemplate": "WR_NAME_Ethiopia_${StartString}_fc",
+ "DataFileTemplate": "deposition_srcs_allregions_${StartString}.nc",
+ "TimeExpectedAvailable": "0800",
+ "ProcessPreJob": "process_pre_job_server_download",
+ "ProcessInJob": "process_in_job_dep",
+ "ProcessEWSPlotting": "process_EWS_plotting_dep",
+ "EWS-Plotting": {
+ "SysConfig": "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/sys/SYS_CONFIG_PINE.json",
+ "NameExtractionConfig": "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/name_extraction/NAME_EXTRACTION_CONFIG.json",
+ "RunConfig": "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/deposition/RUN_CONFIG_DEPO.json",
+ "RunConfigNorm": "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/deposition/RUN_CONFIG_DEPO_NORMALIZED.json",
+ "EastAfrica": {
+ "ChartConfig": "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
+ "Normalize": "False"
+ },
+ "Ethiopia": {
+ "ChartConfig": "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_ETHIOPIA_PINE.json",
+ "Normalize": "True"
+ }
+ }
+}
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/env_suit_config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/env_suit_config_EastAfrica_fc_live.json
new file mode 100644
index 0000000..889190a
--- /dev/null
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/env_suit_config_EastAfrica_fc_live.json
@@ -0,0 +1,92 @@
+{
+ "ServerPathTemplate" : "/storage/sftp/metofficeupload/upload/Ethiopia/fromMO/daily_name/",
+ "InputFileTemplate" : "WR_EnvSuit_Met_Ethiopia_${StartString}_fc",
+ "TimeExpectedAvailable" : "0800",
+ "ProcessPreJob" : "process_pre_job_server_download",
+ "ProcessInJob" : "process_in_job_env2_0",
+ "ProcessEWSPlotting" : "process_EWS_plotting_env2_0",
+ "RESOURCES_PATH": "../../test_data/test_deployment/regions/EastAfrica/resources/met_extractor/",
+ "WORK_PATH" : "../../test_data/test_deployment/regions/EastAfrica/workspace/",
+ "INPUT_PATH" : "../../test_data/test_deployment/regions/EastAfrica/workspace/",
+ "OUTPUT_PATH" : "../../test_data/test_deployment/regions/EastAfrica/workspace/",
+ "SYS_CONFIG" : "../../test_data/test_deployment/regions/EastAfrica/resources/met_extractor/configs/SYS_CONFIG_PINE.json",
+ "FIELD_NAME_CONSTANTS" : "../../test_data/test_deployment/regions/EastAfrica/resources/met_extractor/configs/FIELD_NAME_CONSTANTS.csv",
+ "RUN_TYPE" : "operational",
+ "EXTRACTION_DAYS": 7,
+ "FORECAST_DAYS": 6,
+ "STRAINS": ["LeafRust","StemRust","StripeRust","LeafRust_TempOnly","StemRust_TempOnly","StripeRust_TempOnly"],
+ "PARAMS": {
+ "LeafRust": {
+ "suitability_modules": ["semibool_dewperiod"],
+ "past_steps": 0,
+ "future_steps": 2,
+ "thresholds": {
+ "temperature": [2,15,20,30],
+ "precipitation": 0,
+ "relative_humidity": 90
+ }
+ },
+ "StemRust": {
+ "suitability_modules": ["semibool_dewperiod"],
+ "past_steps": 0,
+ "future_steps": 2,
+ "thresholds": {
+ "temperature": [2,15,24,30],
+ "precipitation": 0,
+ "relative_humidity": 90
+ }
+ },
+ "StripeRust": {
+ "suitability_modules": ["v1_dewperiod"],
+ "past_steps": 0,
+ "future_steps": 7,
+ "thresholds": {
+ "temperature": [2.37,19.8],
+ "precipitation": 0,
+ "relative_humidity": 90
+ }
+ },
+ "LeafRust_TempOnly": {
+ "suitability_modules": ["semibool_dewperiod"],
+ "past_steps": 0,
+ "future_steps": 2,
+ "thresholds": {
+ "temperature": [2,15,20,30],
+ "precipitation": -1,
+ "relative_humidity": 90
+ }
+ },
+ "StemRust_TempOnly": {
+ "suitability_modules": ["semibool_dewperiod"],
+ "past_steps": 0,
+ "future_steps": 2,
+ "thresholds": {
+ "temperature": [2,15,24,30],
+ "precipitation": -1,
+ "relative_humidity": 90
+ }
+ },
+ "StripeRust_TempOnly": {
+ "suitability_modules": ["v1_dewperiod"],
+ "past_steps": 0,
+ "future_steps": 7,
+ "thresholds": {
+ "temperature": [2.37,19.8],
+ "precipitation": -1,
+ "relative_humidity": 90
+ }
+ }
+ },
+ "EWS-Plotting" : {
+ "SysConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/sys/SYS_CONFIG_PINE.json",
+ "RunConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/env_suit/RUN_CONFIG_ENV.json",
+ "EastAfrica" : {
+ "ChartConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_EAST_AFRICA_PINE.json",
+ "FilterForCountry" : "False"
+ },
+ "Ethiopia" : {
+ "ChartConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_ETHIOPIA_PINE.json",
+ "FilterForCountry" : "True"
+ }
+ }
+}
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/epi_config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/epi_config_EastAfrica_fc_live.json
new file mode 100644
index 0000000..c7c4a78
--- /dev/null
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/epi_config_EastAfrica_fc_live.json
@@ -0,0 +1,90 @@
+{
+ "DiseaseNames" : ["StemRust"],
+ "CalculationSpanDays" : [0,1],
+ "TimeStep_hours": "3",
+ "ProcessPreJob" : "process_pre_job_epi",
+ "ProcessInJob" : "process_in_job_epi",
+ "ProcessEWSPlotting" : "process_EWS_plotting_epi",
+ "Host" : {
+ "MaxFieldsPerCell" : "1",
+ "TargetRaster" : "../../test_data/test_deployment/regions/EastAfrica/resources/epimodel/assets/wheat_area_frac_MapSPAM2010_EastAfrica_clipped.tif",
+ "HostRasters" : {
+ "201001010000" : "../../test_data/test_deployment/regions/EastAfrica/resources/epimodel/assets/wheat_area_frac_MapSPAM2010_EastAfrica_clipped.tif"
+ }
+ },
+ "Deposition" : {
+ "VariableNames" : ["P_GRAMINIS_DEPOSITION"],
+ "PathTemplate" : "${WorkspacePath}DEPOSITION_${DateString}/WR_NAME_Ethiopia_${DateString}_fc/",
+ "SuccessFileTemplate" : "${WorkspacePath}DEPOSITION_${StartString}/STATUS_SUCCESS",
+ "FileListerFunction" : "list_onefile_operational",
+ "FileNameTemplate" : "deposition_srcs_allregions_${DateString}.nc",
+ "FileNamePrepared" : "?"
+ },
+ "Environment" : {
+ "PathTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${DateString}/processed/${RegionName}/${DiseaseName}/",
+ "SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS",
+ "FileListerFunction" : "list_onefile_operational",
+ "FileNameTemplate" : "RIE_value.nc",
+ "FileNamePrepared" : "?"
+ },
+ "Env" : {
+ "SuccessFileTemplate" : "${WorkspacePath}EPI_${YesterdayString}/STATUS_SUCCESS",
+ "PathTemplate" : "${WorkspacePath}EPI_${YesterdayString}/${RegionName}/${DiseaseName}/",
+ "FileNameTemplate" : "infections_config_EastAfrica_fc_live_Epidemiology_*_env_progression.csv",
+ "FileListerFunction" : "list_onefile_historical",
+ "FileLoaderFunction" : "load_and_restructure_epi_file",
+ "FileNamePrepared" : "?"
+ },
+ "ps" : {
+ "SuccessFileTemplate" : "${WorkspacePath}EPI_${YesterdayString}/STATUS_SUCCESS",
+ "PathTemplate" : "${WorkspacePath}EPI_${YesterdayString}/${RegionName}/${DiseaseName}/",
+ "FileNameTemplate" : "infections_config_EastAfrica_fc_live_Epidemiology_*_psbeta0.004gamma0.00025alpha1.0_progression.csv",
+ "FileListerFunction" : "list_onefile_historical",
+ "FileLoaderFunction" : "load_and_restructure_epi_file",
+ "FileNamePrepared" : "?"
+ },
+ "Epi" : [
+ {
+ "model" : "Env",
+ "modelArguments" : {},
+ "infectionRasterFileName" : "?",
+ "description": "env. suitability",
+ "rescale_output_by_host_raster": false,
+ "analysis" : {
+ "vmin" : 0.0e+0,
+ "vmax" : 1.5e+1,
+ "subplot_position" : [0,0],
+ "cmapString" : "CMRmap_r",
+ "bounds" : [32.5,48.0,3.3,15.0],
+ "UTMprojection" : 37
+ }
+ },{
+ "model" : "ps",
+ "modelArguments": {
+ "beta": 4.0e-3,
+ "gamma": 2.5e-4,
+ "alpha": 1.0,
+ "infection": "previous"
+ },
+ "infectionRasterFileName" : "?",
+ "description": "SE[beta*log10(alpha*D)+gamma*P]",
+ "rescale_output_by_host_raster": true,
+ "analysis" : {
+ "vmin" : 0.0e+0,
+ "vmax" : 1,
+ "subplot_position" : [0,1],
+ "cmapString" : "CMRmap_r",
+ "bounds": [32.5,48.0,3.3,15.0],
+ "UTMprojection" : 37
+ }
+ }
+ ],
+ "EWS-Plotting" : {
+ "SysConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/sys/SYS_CONFIG_PINE.json",
+ "RunConfig_seasonsofar" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/epi/RUN_CONFIG_EPI.json",
+ "RunConfig_seasonplusforecast" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/epi/RUN_CONFIG_EPI.json",
+ "ChartConfig" : "../../test_data/test_deployment/regions/EastAfrica/resources/plotting/configs/chart/CHART_CONFIG_ETHIOPIA_PINE.json",
+ "PlottingRegionName" : "Ethiopia",
+ "EpiCase" : "ps"
+ }
+}
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/survey_config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/survey_config_EastAfrica_fc_live.json
new file mode 100644
index 0000000..b134251
--- /dev/null
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/survey_config_EastAfrica_fc_live.json
@@ -0,0 +1,40 @@
+{
+ "ProcessPreJob" : "process_pre_job_survey",
+ "ProcessInJob" : "process_in_job_survey",
+ "ProcessEWSPlotting": "process_EWS_plotting_survey",
+ "AcceptableDowntimeDays": 70,
+ "SeasonStartString" : "20220930",
+ "SkipServerDownload" : true,
+ "ServerCredentialsFile" : "../../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/Cred-ODK-EIAR.json",
+ "ServerPathExtra" : "OVERRIDDEN",
+ "FormEdits" : {
+ "wheat_rust_survey_1_0" : {
+ "add" : {
+ "Origin" : "ODK-server",
+ "PublishedLevel" : "Raw"
+
+ }
+ },
+ "akpyJHvYxkLKPkxFJnPyTW" : {
+ "add" : {
+ "Origin" : "kobo-server",
+ "PublishedLevel" : "Raw"
+ },
+ "filter_by_list": {
+ "surveyor_infromation-country" : ["Kenya", "Ethiopia"]
+ }
+ },
+ "WRSIS" : {}
+ },
+ "Groups" : {
+ "PROD" : {
+ "Origin" : ["ODK-server", "kobo-server"]
+ }
+ },
+ "GroupBy" : ["Origin"],
+ "GroupsToIgnore" : ["ODK-server", "kobo-server", "newODK", "newODK2", "CSV-CAM"],
+ "SurveyFormat" : "ODK",
+ "SurveyorNameCol" : "surveyor_infromation-surveyor_name",
+ "SourcesRegionName" : "EastAfrica",
+ "SourcesConfigDefault" : "../../test_data/test_deployment/regions/EastAfrica/resources/source_gen/configs/config_EastAfrica_mapspam2017.json"
+}
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/sys_config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/sys_config_EastAfrica_fc_live.json
new file mode 100644
index 0000000..a47eb7a
--- /dev/null
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/sys_config_EastAfrica_fc_live.json
@@ -0,0 +1,9 @@
+{
+ "RegionName" : "EastAfrica",
+ "SubRegionNames" : ["EastAfrica","Ethiopia"],
+ "StartTime" : "?",
+ "StartString" : "?",
+ "WorkspacePathout" : "set_in_the_code",
+ "WorkspacePath" : "set_in_the_code",
+ "ResourcesPath" : "../../test_data/test_deployment/regions/EastAfrica/resources/"
+}
--
GitLab