diff --git a/ProcessorComponents.py b/ProcessorComponents.py index 86353e16f4c72f87541b804f2f5acc6a33a65a8c..49842234f362b40eba4636cccb7b6759b515b323 100644 --- a/ProcessorComponents.py +++ b/ProcessorComponents.py @@ -149,11 +149,11 @@ def query_proceed(necessary_file,description): assert os.path.isfile(necessary_file) - logger.info(f"Found:\n{necessary_file}\nso {description} job has succeeded for this date, EPI shall run.") + logger.info(f"Found:\n{necessary_file}\nso {description} job has succeeded for this date, this job shall run.") except AssertionError as e: - logger.info(f"Failed to find:\n{necessary_file}\nso {description} job has not yet succeeded for this date, so cannot run EPI.") + logger.info(f"Failed to find:\n{necessary_file}\nso {description} job has not yet succeeded for this date, so cannot run this job.") endScript(premature=True) @@ -161,11 +161,13 @@ def query_proceed(necessary_file,description): return True -def process_pre_job_epi(input_args): - '''Returns a boolean as to whether the job is ready for full processing.''' - - logger.info('started process_pre_job_epi()') +def query_past_successes(input_args): + '''Checks if deposition and environment jobs are already completed + successfully. If not, it raises an error.''' + + component = input_args.component + # check configs can be loaded config_fns = input_args.config_paths for configFile in config_fns: try: @@ -174,25 +176,45 @@ def process_pre_job_epi(input_args): logger.exception(f"Failure in opening or checking config {configFile}") endScript(premature=True) + # some config initialisation is necessary + config_i['StartString'] = input_args.start_date + # check if deposition data is readily available - dep_success_file = Template(config_i['Epidemiology']['Deposition']['SuccessFileTemplate']).substitute(**config_i) + dep_success_file = Template(config_i[component]['Deposition']['SuccessFileTemplate']).substitute(**config_i) query_proceed(dep_success_file,'deposition') - env_success_file = Template(config_i['Epidemiology']['Environment']['SuccessFileTemplate']).substitute(**config_i) - query_proceed(env_success_file,'environment') + # check if environment data is readily available + env_success_file = Template(config_i[component]['Environment']['SuccessFileTemplate']).substitute(**config_i) + query_proceed(env_success_file,'envsironment') + + return True + +def process_pre_job_epi(input_args): + '''Returns a boolean as to whether the job is ready for full processing.''' - #determine end time, from config file + logger.info('started process_pre_job_epi()') - arg_start_date = input_args.start_date - calc_span_days = config_i['Epidemiology']['CalculationSpanDays'] - assert len(calc_span_days) == 2 + # check pre-requisite jobs are complete + query_past_successes(input_args) - start_time, end_time = calc_epi_date_range(arg_start_date,calc_span_days) + config_fns = input_args.config_paths - # warn if it is a long timespan - date_diff = end_time - start_time - if date_diff.days > 100: - logger.warning("More than 100 days will be calculated over, likely longer than any single season") + for configFile in config_fns: + + # they should be working if the script made it this far, no need to try + config_i = open_and_check_config(configFile) + + #determine end time, from config file + arg_start_date = input_args.start_date + calc_span_days = config_i['Epidemiology']['CalculationSpanDays'] + assert len(calc_span_days) == 2 + + start_time, end_time = calc_epi_date_range(arg_start_date,calc_span_days) + + # warn if it is a long timespan + date_diff = end_time - start_time + if date_diff.days > 100: + logger.warning("More than 100 days will be calculated over, likely longer than any single season") return True