diff --git a/configs/docker/run/launchAndRunDockerEWS.sh b/configs/docker/run/launchAndRunDockerEWS.sh
index cf20d14f89f0d8dede65a6e50098a24f7adcdef0..7e62a042cafe385e0b18c5d2d7e0d898c14066d7 100755
--- a/configs/docker/run/launchAndRunDockerEWS.sh
+++ b/configs/docker/run/launchAndRunDockerEWS.sh
@@ -5,7 +5,7 @@ upload=''
islive=''
clearup=''
component='not_set'
-loglevel='not_set'
+loglevel='info'
config='/storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json' # default
rundate=$(date '+%Y%m%d') # default today
diff --git a/ews/coordinator/processor_advisory.py b/ews/coordinator/processor_advisory.py
index 78aaa13e6110342e3eeeafe8f929f6a6d809f2a8..c6fcd9b8ad836c05673db36ff4a49e2189619e62 100644
--- a/ews/coordinator/processor_advisory.py
+++ b/ews/coordinator/processor_advisory.py
@@ -5,9 +5,9 @@ import logging
# gitlab projects
# TODO: Package these projects so they are robust for importing
-from ews.coordinator.utils.processor_utils import short_name
from ews.coordinator.processor_base import ProcessorBase
from ews.advisory_builder import data_gatherer
+from ews.coordinator.utils.processor_utils import short_name
class ProcessorAdvisory(ProcessorBase):
@@ -67,4 +67,4 @@ class ProcessorAdvisory(ProcessorBase):
if __name__ == '__main__':
processor = ProcessorAdvisory()
- processor.run_processor("Advisory")
+ processor.run_processor("Advisory", "ADVISORY")
diff --git a/ews/coordinator/processor_base.py b/ews/coordinator/processor_base.py
index e0c93a6de934a819071e55732ff71c1df515dfa5..cb558f308f6911b5de3efccb78a114b6c16434d5 100755
--- a/ews/coordinator/processor_base.py
+++ b/ews/coordinator/processor_base.py
@@ -21,7 +21,7 @@ from typing import List
from ews.coordinator.utils import processor_utils
from ews.coordinator.utils.jobstatus import Jobstatus
-from ews.coordinator.utils.processor_utils import short_name, open_and_check_config, end_script, end_job, append_item_to_list, \
+from ews.coordinator.utils.processor_utils import open_and_check_config, end_script, end_job, append_item_to_list, \
clear_up
@@ -62,14 +62,14 @@ class ProcessorBase:
# positional arguments do not start with - or -- and are always required
# optional arguments start with - or -- and default is required = False
- my_parser.add_argument(
- '-p', '--component',
- type = str,
- choices = list(short_name.keys()),
- required = False,
- dest = 'component',
- help = '''Name of EWS component to process, which must be present
- in the config file.''')
+ # my_parser.add_argument(
+ # '-p', '--component',
+ # type = str,
+ # choices = list(short_name.keys()),
+ # required = False,
+ # dest = 'component',
+ # help = '''Name of EWS component to process, which must be present
+ # in the config file.''')
my_parser.add_argument(
'-c', '--config',
@@ -223,6 +223,7 @@ class ProcessorBase:
config_paths: List[str] = args['config_paths']
component: str = args['component']
+ short_name: str = args['short_name']
start_date: str = args['start_date']
clearup: bool = args['clearup']
@@ -232,7 +233,7 @@ class ProcessorBase:
# determine job directory
workspacePath = universal_config['WorkspacePathout']
- job_path: str = f'{workspacePath}{short_name[component]}_{start_date}'
+ job_path: str = f'{workspacePath}{short_name}_{start_date}'
self.prepare_job_directory(job_path)
is_live: bool = args["live"]
@@ -352,7 +353,7 @@ class ProcessorBase:
if status.is_success() & (clearup is True):
logger.info('Clearing up')
- clearup_dest_dir = f"{workspacePath}/clearup/{short_name[component]}_{start_date}/"
+ clearup_dest_dir = f"{workspacePath}/clearup/{short_name}_{start_date}/"
Path(clearup_dest_dir).mkdir(parents = True, exist_ok = True)
logger.info(f"While developing, moving directories to this directory : {clearup_dest_dir}")
@@ -373,12 +374,13 @@ class ProcessorBase:
def process_post_job(self, jobPath, configjson):
raise NotImplementedError
- def run_processor(self, component: str):
+ def run_processor(self, component: str, shortname: str):
logger.info("Make sure to `conda activate py3EWSepi` environment!")
logger.info("Make sure that flagdir package is available (on PYTHONPATH)")
try:
args: dict = self.parse_and_check_args()
args["component"] = component
+ args["short_name"] = shortname
self.run_process(args)
diff --git a/ews/coordinator/processor_deposition.py b/ews/coordinator/processor_deposition.py
index 48602331bc41e6a7043ce0841bc0b7aa33f37cd7..ce802613d7ae291a11581b70a9b76d58e5b306fa 100644
--- a/ews/coordinator/processor_deposition.py
+++ b/ews/coordinator/processor_deposition.py
@@ -178,5 +178,5 @@ if __name__ == '__main__':
print(__name__)
print(processor_utils.__name__)
processor = ProcessorDeposition()
- processor.run_processor("Deposition")
+ processor.run_processor("Deposition", "DEPOSITION")
diff --git a/ews/coordinator/processor_environment.py b/ews/coordinator/processor_environment.py
index 46d5d91aca7222c24cc2f44b3bf0191dd5c78760..9dfd8c28291d2db0641f52987854441c5a79e8f9 100644
--- a/ews/coordinator/processor_environment.py
+++ b/ews/coordinator/processor_environment.py
@@ -369,4 +369,4 @@ class ProcessorEnvironment(ProcessorBase):
if __name__ == '__main__':
processor = ProcessorEnvironment()
- processor.run_processor("Environment")
+ processor.run_processor("Environment", "ENVIRONMENT_2.0")
diff --git a/ews/coordinator/processor_epidemiology.py b/ews/coordinator/processor_epidemiology.py
index 5418a4cd6aa985b6de53de0605113a70b930fa0b..9dccc3374db8d91cbea7e377f1de37175836ba27 100644
--- a/ews/coordinator/processor_epidemiology.py
+++ b/ews/coordinator/processor_epidemiology.py
@@ -30,8 +30,7 @@ from ews.coordinator.utils.processor_utils import (
open_and_check_config,
get_only_existing_globs,
end_job,
- short_name,
- disease_latin_name_dict
+ disease_latin_name_dict, short_name
)
logger = logging.getLogger(__name__)
@@ -788,4 +787,4 @@ class ProcessorEpidemiology(ProcessorBase):
if __name__ == '__main__':
processor = ProcessorEpidemiology()
- processor.run_processor("Epidemiology")
\ No newline at end of file
+ processor.run_processor("Epidemiology", "EPI")
diff --git a/ews/coordinator/processor_scraper.py b/ews/coordinator/processor_scraper.py
index 23ab96bad502586098b07f0238d510b197487108..41f82439168f3a6e58080e88b2542eec74eb1117 100644
--- a/ews/coordinator/processor_scraper.py
+++ b/ews/coordinator/processor_scraper.py
@@ -465,4 +465,4 @@ class ProcessorScraper(ProcessorBase):
if __name__ == '__main__':
processor = ProcessorScraper()
- processor.run_processor("Scraper")
+ processor.run_processor("Scraper", "SCRAPER")
diff --git a/ews/coordinator/processor_surveys.py b/ews/coordinator/processor_surveys.py
index 30ab67b60bee5a2512139fe7fae2cb581fdd1933..dbc667a7cb796407ec88df6263aeb49d9b50f804 100644
--- a/ews/coordinator/processor_surveys.py
+++ b/ews/coordinator/processor_surveys.py
@@ -424,4 +424,4 @@ class ProcessorSurveys(ProcessorBase):
if __name__ == '__main__':
print(__name__)
processor = ProcessorSurveys()
- processor.run_processor("Survey")
+ processor.run_processor("Survey", "SURVEYDATA")
diff --git a/ews/coordinator/utils/processor_utils.py b/ews/coordinator/utils/processor_utils.py
index 78f90a6cff5c6d60fb23ce6348c0be1710276d71..9b8248d14f6be565372e042c05f477ee3023d902 100644
--- a/ews/coordinator/utils/processor_utils.py
+++ b/ews/coordinator/utils/processor_utils.py
@@ -36,8 +36,7 @@ short_name = {
'Epidemiology' : 'EPI',
'Survey' : 'SURVEYDATA',
'Scraper' : 'SCRAPER',
- 'MetResample' : 'MET_RESAMPLE',
- 'MetPlotting' : 'MetPlotting',
+ 'MetResample' : 'MET_RESAMPLE'
}
disease_latin_name_dict = {
diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py
index 9a02b0ce0c9ca173721431b8030a0c1eb8845d04..340298cfd550ff7869a764b8398c134f8ac1716f 100644
--- a/tests/integration/partial/integration_test_utils.py
+++ b/tests/integration/partial/integration_test_utils.py
@@ -171,6 +171,7 @@ class IntegrationTestUtils:
@staticmethod
def run_partial_integration_test_pipeline(component: str,
+ shortname: str,
start_date: str,
processor: ProcessorBase,
**kwargs):
@@ -181,6 +182,7 @@ class IntegrationTestUtils:
:param processor:
:param component:
+ :param shortname:
:param start_date:
:param kwargs:
:return:
@@ -195,6 +197,7 @@ class IntegrationTestUtils:
args_dict['noupload'] = True
args_dict['start_date'] = start_date
args_dict['component'] = component
+ args_dict['short_name'] = shortname
args_dict['config_paths'] = config_paths
args_dict['log_level'] = 'info'
args_dict['clearup'] = True
diff --git a/tests/integration/partial/test_advisory.py b/tests/integration/partial/test_advisory.py
index 2ca6ef8ce0e155294d458783a5f3714a952fa983..d72450e0663ace7f33c026f8e024e5ab2cfa16bc 100644
--- a/tests/integration/partial/test_advisory.py
+++ b/tests/integration/partial/test_advisory.py
@@ -69,10 +69,13 @@ class TestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
@staticmethod
def run_advisory_pipeline():
component = 'Advisory'
+ short_name = 'ADVISORY'
advisory_processor = ProcessorAdvisory()
IntegrationTestUtils.run_partial_integration_test_pipeline(component,
+ short_name,
IntegrationTestUtils.TEST_START_DATE,
advisory_processor)
+
if __name__ == '__main__':
unittest.main()
diff --git a/tests/integration/partial/test_deposition.py b/tests/integration/partial/test_deposition.py
index f8f214c14737816bcb7834152d131cec44422c76..f4d9b20c70052928a6c658786ec4273b000cdfc1 100644
--- a/tests/integration/partial/test_deposition.py
+++ b/tests/integration/partial/test_deposition.py
@@ -64,10 +64,12 @@ class TestDeposition(BaseDepoTestSuite.DepoTestSuite):
@staticmethod
def run_depo_pipeline():
component = 'Deposition'
+ short_name = 'DEPOSITION'
# need EMAIL_CRED in the environment before we create a Processor
os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
deposition_processor = ProcessorDeposition()
IntegrationTestUtils.run_partial_integration_test_pipeline(component,
+ short_name,
IntegrationTestUtils.TEST_START_DATE,
deposition_processor)
diff --git a/tests/integration/partial/test_env_suit.py b/tests/integration/partial/test_env_suit.py
index 38bde21354d1812c7f9fab4c6750e0f68a5fcd69..9ade40d8496dcce1be9080ac360d3243e7913b2d 100644
--- a/tests/integration/partial/test_env_suit.py
+++ b/tests/integration/partial/test_env_suit.py
@@ -67,10 +67,12 @@ class TestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
@staticmethod
def run_env_pipeline():
component = 'Environment'
+ short_name = 'ENVIRONMENT_2.0'
# need EMAIL_CRED in the environment before we create a Processor
os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
environment_processor = ProcessorEnvironment()
IntegrationTestUtils.run_partial_integration_test_pipeline(component,
+ short_name,
IntegrationTestUtils.TEST_START_DATE,
environment_processor)
diff --git a/tests/integration/partial/test_epi.py b/tests/integration/partial/test_epi.py
index 65837610098004fbd666f1cbfac8cbef9deeebbd..b9466d63956e6e5c00cadd3b5bc3e35a09fa1c39 100644
--- a/tests/integration/partial/test_epi.py
+++ b/tests/integration/partial/test_epi.py
@@ -67,10 +67,12 @@ class TestEpi(BaseEpiTestSuite.EpiTestSuite):
@staticmethod
def run_epi_pipeline():
component = 'Epidemiology'
+ short_name = 'EPI'
# need EMAIL_CRED in the environment before we create a Processor
os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
epi_processor = ProcessorEpidemiology()
IntegrationTestUtils.run_partial_integration_test_pipeline(component,
+ short_name,
IntegrationTestUtils.TEST_START_DATE,
epi_processor)
diff --git a/tests/integration/partial/test_survey.py b/tests/integration/partial/test_survey.py
index 0d7c2f881b3e27d97d6fe13e51d1bb4bdc212f62..52edb0f0c663edbfe2a832b9bc23a295185b681a 100644
--- a/tests/integration/partial/test_survey.py
+++ b/tests/integration/partial/test_survey.py
@@ -65,8 +65,10 @@ class TestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
@staticmethod
def run_survey_pipeline():
component = 'Survey'
+ short_name = 'SURVEYDATA'
survey_processor = ProcessorSurveys()
IntegrationTestUtils.run_partial_integration_test_pipeline(component,
+ short_name,
IntegrationTestUtils.TEST_START_DATE,
survey_processor)