diff --git a/ews/coordinator/processor_base.py b/ews/coordinator/processor_base.py
index b21782cd1462af1f6f75243a1b760c048392126b..230c71a78d8ef3d6c54b29d3fd787e16348eb931 100755
--- a/ews/coordinator/processor_base.py
+++ b/ews/coordinator/processor_base.py
@@ -17,6 +17,7 @@ or::
$ ./run_Processor.sh -p Deposition --islive --config config_Nepal_template.json --noupload -s 20200715
'''
from abc import abstractmethod, ABCMeta
+from argparse import ArgumentParser
from typing import List
from ews.coordinator.utils import processor_utils
@@ -52,7 +53,7 @@ class ProcessorBase:
def parse_and_check_args(self) -> dict:
# define the command line arguments
- my_parser = argparse.ArgumentParser(
+ my_parser: ArgumentParser = argparse.ArgumentParser(
description = 'Command-line arguments for coordinator script of env suitability model')
# Add the arguments
@@ -114,6 +115,8 @@ class ProcessorBase:
dest = 'clearup',
)
+ self.add_additional_command_line_args(my_parser)
+
# get an object holding all of the args
args = my_parser.parse_args()
@@ -140,6 +143,15 @@ class ProcessorBase:
dictionary: dict = vars(args)
return dictionary
+ def add_additional_command_line_args(self, arg_parser: ArgumentParser) -> None:
+ """
+ override this method to add additional command line arguments
+
+ :param arg_parser:
+ :return:
+ """
+ pass
+
@staticmethod
def generate_job_directory_path(config: dict) -> str:
"""
diff --git a/ews/coordinator/processor_deposition.py b/ews/coordinator/processor_deposition.py
index b145c7e35d67c0d71deafe1790c596e0e1d3fd97..4ade43b42a8d57fcee8e61efb98a1707760943e7 100644
--- a/ews/coordinator/processor_deposition.py
+++ b/ews/coordinator/processor_deposition.py
@@ -3,6 +3,8 @@
import logging
import os
+import datetime
+from argparse import ArgumentParser
from glob import glob
from pathlib import Path
from string import Template
@@ -43,6 +45,17 @@ class ProcessorDeposition(ProcessorBase):
def process_in_job_dep(self, job_path: str, status: Jobstatus, config: dict, component: str):
+
+ """
+ copies and untars the deposition data - ready for plotting
+
+ :param job_path:
+ :param status:
+ :param config:
+ :param component:
+ :return:
+ """
+
logger.info('started process_in_job_dep()')
file_path = Template(config['ServerPathTemplate']).substitute(**config)
@@ -98,8 +111,7 @@ class ProcessorDeposition(ProcessorBase):
return proc_out
- def process_EWS_plotting_dep(self, job_path, config) -> [str]:
- '''Returns a list of output files for transfer.'''
+ def process_EWS_plotting_dep(self, job_path: str, config: dict) -> [str]:
logger.info('started process_EWS_plotting_dep()')
@@ -115,18 +127,22 @@ class ProcessorDeposition(ProcessorBase):
deposition_data_file_name = Template(config['DataFileTemplate']).substitute(**config)
name_file_wildcard = f"{deposition_path}/{deposition_data_file_name}"
+ wheat_group = config['wheat_group']
+ wheat_group_display = config['wheat_group_display']
+ sys_config = config['EWS-Plotting']['SysConfig']
+ name_extraction_config = config['EWS-Plotting']['NameExtractionConfig']
+ run_config = config['EWS-Plotting']['RunConfig']
+ run_config_norm = config['EWS-Plotting']['RunConfigNorm']
+ run_date_string = config['StartString']
+
ews_plotting_output_globs = []
for region in regions:
output_dir = f"{job_path}/plotting/{region.lower()}"
- Path(output_dir).mkdir(parents=True, exist_ok=True)
+ Path(output_dir).mkdir(parents = True, exist_ok = True)
- sys_config = config['EWS-Plotting']['SysConfig']
- name_extraction_config = config['EWS-Plotting']['NameExtractionConfig']
- run_config = config['EWS-Plotting']['RunConfig']
- run_config_norm = config['EWS-Plotting']['RunConfigNorm']
chart_config = config['EWS-Plotting'][region]['ChartConfig']
normalize = config['EWS-Plotting'][region]['Normalize']
extraction_file_prefix = 'deposition_' + region.lower()
@@ -144,12 +160,14 @@ class ProcessorDeposition(ProcessorBase):
name_file_wildcard_arg = name_file_wildcard,
wheat_sources_dir_arg = deposition_path,
output_dir_arg = output_dir,
- issue_date_arg = config['StartString'],
+ issue_date_arg = run_date_string,
extraction_file_prefix_arg = extraction_file_prefix)
# asia/east africa env suit should not perform normalization, false gets passed here for these areas
depo_processor.name_extract_params.NORMALIZE = (normalize.upper() == "TRUE")
+ depo_processor.depo_run_params.WHEAT_GROUP = wheat_group
+ depo_processor.depo_run_params.WHEAT_GROUP_DISPLAY = wheat_group_display
depo_processor.process()
@@ -170,6 +188,24 @@ class ProcessorDeposition(ProcessorBase):
return ews_plotting_outputs
+ def add_additional_command_line_args(self, my_parser: ArgumentParser) -> None:
+ my_parser.add_argument(
+ '-wg', '--wheat_group',
+ metavar = 'string',
+ type = str,
+ dest = 'wheat_group',
+ required = True,
+ help = '''the survey group used when creating the source data for the deposition run''')
+
+ my_parser.add_argument(
+ '-wgd', '--wheat_group_display_name',
+ metavar = 'string',
+ type = str,
+ dest = 'wheat_group_display',
+ required = False,
+ help = '''a value to display in the plots for the wheat group - if unspecified will not appear''')
+
+
if __name__ == '__main__':
print(__name__)
print(processor_utils.__name__)
diff --git a/ews/coordinator/utils/processor_utils.py b/ews/coordinator/utils/processor_utils.py
index 3abaaba0f4a3f613563d853e108f5062d7d0f0a3..5d98c4c316589f6ed2ab58b02e8fb1d461fb6079 100644
--- a/ews/coordinator/utils/processor_utils.py
+++ b/ews/coordinator/utils/processor_utils.py
@@ -72,7 +72,7 @@ def filter_exception_message(exc_info):
return
-class PasswordODKFilter(logging.Filter):
+class PasswordFilter(logging.Filter):
'''Built with
https://docs.python.org/3.6/howto/logging-cookbook.html#configuring-filters-with-dictconfig
'''
@@ -97,11 +97,12 @@ class PasswordODKFilter(logging.Filter):
return True
-def add_filters_to_sublogger(logger_case):
- for h in logger_case.handlers:
-
- h.addFilter(PasswordODKFilter())
+# def add_filters_to_sublogger(logger_case):
+#
+# for h in logger_case.handlers:
+#
+# h.addFilter(PasswordODKFilter())
def open_and_check_config(configFile) -> dict:
@@ -401,7 +402,7 @@ def setup_logging(job_file_path: str,
log_config_dict['formatters']['detailed']['format'] = \
log_config_dict['formatters']['detailed']['format'].replace("{arg_string}", arg_string)
- log_config_dict['filters']['mask_passwords']['()'] = PasswordODKFilter
+ log_config_dict['filters']['mask_passwords']['()'] = PasswordFilter
log_config_dict['handlers']['handler_job']['filename'] = job_file_path
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/depo_config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/depo_config_EastAfrica_fc_live.json
index a62bd136b87387a96c9165e93913c9c146370131..4c63c00844be2d64a5c0518bd9069f8de2779f8d 100644
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/depo_config_EastAfrica_fc_live.json
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/configs/coordinator/depo_config_EastAfrica_fc_live.json
@@ -1,6 +1,6 @@
{
"ServerPathTemplate": "OVERRIDDEN",
- "InputFileTemplate": "WR_NAME_Ethiopia_${StartString}_fc",
+ "InputFileTemplate": "WR_NAME_Ethiopia_*_${StartString}_fc",
"DataFileTemplate": "deposition_srcs_allregions_${StartString}.nc",
"EWS-Plotting": {
"SysConfig": "${ProjectRoot}/regions/EastAfrica/resources/configs/coordinator/sys_config_EastAfrica_fc_live.json",