From c2dd29b673752f49a72bf56ea73f7da1c858e553 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Thu, 30 May 2024 11:34:57 +0100
Subject: [PATCH] making processor components pass their shortname in - rather
than relying on the short_name map to know about all components
---
configs/docker/run/launchAndRunDockerEWS.sh | 2 +-
ews/coordinator/processor_advisory.py | 4 +--
ews/coordinator/processor_base.py | 26 ++++++++++---------
ews/coordinator/processor_deposition.py | 2 +-
ews/coordinator/processor_environment.py | 2 +-
ews/coordinator/processor_epidemiology.py | 5 ++--
ews/coordinator/processor_scraper.py | 2 +-
ews/coordinator/processor_surveys.py | 2 +-
ews/coordinator/utils/processor_utils.py | 3 +--
.../partial/integration_test_utils.py | 3 +++
tests/integration/partial/test_advisory.py | 3 +++
tests/integration/partial/test_deposition.py | 2 ++
tests/integration/partial/test_env_suit.py | 2 ++
tests/integration/partial/test_epi.py | 2 ++
tests/integration/partial/test_survey.py | 2 ++
15 files changed, 38 insertions(+), 24 deletions(-)
diff --git a/configs/docker/run/launchAndRunDockerEWS.sh b/configs/docker/run/launchAndRunDockerEWS.sh
index cf20d14..7e62a04 100755
--- a/configs/docker/run/launchAndRunDockerEWS.sh
+++ b/configs/docker/run/launchAndRunDockerEWS.sh
@@ -5,7 +5,7 @@ upload=''
islive=''
clearup=''
component='not_set'
-loglevel='not_set'
+loglevel='info'
config='/storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json' # default
rundate=$(date '+%Y%m%d') # default today
diff --git a/ews/coordinator/processor_advisory.py b/ews/coordinator/processor_advisory.py
index 78aaa13..c6fcd9b 100644
--- a/ews/coordinator/processor_advisory.py
+++ b/ews/coordinator/processor_advisory.py
@@ -5,9 +5,9 @@ import logging
# gitlab projects
# TODO: Package these projects so they are robust for importing
-from ews.coordinator.utils.processor_utils import short_name
from ews.coordinator.processor_base import ProcessorBase
from ews.advisory_builder import data_gatherer
+from ews.coordinator.utils.processor_utils import short_name
class ProcessorAdvisory(ProcessorBase):
@@ -67,4 +67,4 @@ class ProcessorAdvisory(ProcessorBase):
if __name__ == '__main__':
processor = ProcessorAdvisory()
- processor.run_processor("Advisory")
+ processor.run_processor("Advisory", "ADVISORY")
diff --git a/ews/coordinator/processor_base.py b/ews/coordinator/processor_base.py
index e0c93a6..cb558f3 100755
--- a/ews/coordinator/processor_base.py
+++ b/ews/coordinator/processor_base.py
@@ -21,7 +21,7 @@ from typing import List
from ews.coordinator.utils import processor_utils
from ews.coordinator.utils.jobstatus import Jobstatus
-from ews.coordinator.utils.processor_utils import short_name, open_and_check_config, end_script, end_job, append_item_to_list, \
+from ews.coordinator.utils.processor_utils import open_and_check_config, end_script, end_job, append_item_to_list, \
clear_up
@@ -62,14 +62,14 @@ class ProcessorBase:
# positional arguments do not start with - or -- and are always required
# optional arguments start with - or -- and default is required = False
- my_parser.add_argument(
- '-p', '--component',
- type = str,
- choices = list(short_name.keys()),
- required = False,
- dest = 'component',
- help = '''Name of EWS component to process, which must be present
- in the config file.''')
+ # my_parser.add_argument(
+ # '-p', '--component',
+ # type = str,
+ # choices = list(short_name.keys()),
+ # required = False,
+ # dest = 'component',
+ # help = '''Name of EWS component to process, which must be present
+ # in the config file.''')
my_parser.add_argument(
'-c', '--config',
@@ -223,6 +223,7 @@ class ProcessorBase:
config_paths: List[str] = args['config_paths']
component: str = args['component']
+ short_name: str = args['short_name']
start_date: str = args['start_date']
clearup: bool = args['clearup']
@@ -232,7 +233,7 @@ class ProcessorBase:
# determine job directory
workspacePath = universal_config['WorkspacePathout']
- job_path: str = f'{workspacePath}{short_name[component]}_{start_date}'
+ job_path: str = f'{workspacePath}{short_name}_{start_date}'
self.prepare_job_directory(job_path)
is_live: bool = args["live"]
@@ -352,7 +353,7 @@ class ProcessorBase:
if status.is_success() & (clearup is True):
logger.info('Clearing up')
- clearup_dest_dir = f"{workspacePath}/clearup/{short_name[component]}_{start_date}/"
+ clearup_dest_dir = f"{workspacePath}/clearup/{short_name}_{start_date}/"
Path(clearup_dest_dir).mkdir(parents = True, exist_ok = True)
logger.info(f"While developing, moving directories to this directory : {clearup_dest_dir}")
@@ -373,12 +374,13 @@ class ProcessorBase:
def process_post_job(self, jobPath, configjson):
raise NotImplementedError
- def run_processor(self, component: str):
+ def run_processor(self, component: str, shortname: str):
logger.info("Make sure to `conda activate py3EWSepi` environment!")
logger.info("Make sure that flagdir package is available (on PYTHONPATH)")
try:
args: dict = self.parse_and_check_args()
args["component"] = component
+ args["short_name"] = shortname
self.run_process(args)
diff --git a/ews/coordinator/processor_deposition.py b/ews/coordinator/processor_deposition.py
index 4860233..ce80261 100644
--- a/ews/coordinator/processor_deposition.py
+++ b/ews/coordinator/processor_deposition.py
@@ -178,5 +178,5 @@ if __name__ == '__main__':
print(__name__)
print(processor_utils.__name__)
processor = ProcessorDeposition()
- processor.run_processor("Deposition")
+ processor.run_processor("Deposition", "DEPOSITION")
diff --git a/ews/coordinator/processor_environment.py b/ews/coordinator/processor_environment.py
index 46d5d91..9dfd8c2 100644
--- a/ews/coordinator/processor_environment.py
+++ b/ews/coordinator/processor_environment.py
@@ -369,4 +369,4 @@ class ProcessorEnvironment(ProcessorBase):
if __name__ == '__main__':
processor = ProcessorEnvironment()
- processor.run_processor("Environment")
+ processor.run_processor("Environment", "ENVIRONMENT_2.0")
diff --git a/ews/coordinator/processor_epidemiology.py b/ews/coordinator/processor_epidemiology.py
index 5418a4c..9dccc33 100644
--- a/ews/coordinator/processor_epidemiology.py
+++ b/ews/coordinator/processor_epidemiology.py
@@ -30,8 +30,7 @@ from ews.coordinator.utils.processor_utils import (
open_and_check_config,
get_only_existing_globs,
end_job,
- short_name,
- disease_latin_name_dict
+ disease_latin_name_dict, short_name
)
logger = logging.getLogger(__name__)
@@ -788,4 +787,4 @@ class ProcessorEpidemiology(ProcessorBase):
if __name__ == '__main__':
processor = ProcessorEpidemiology()
- processor.run_processor("Epidemiology")
\ No newline at end of file
+ processor.run_processor("Epidemiology", "EPI")
diff --git a/ews/coordinator/processor_scraper.py b/ews/coordinator/processor_scraper.py
index 23ab96b..41f8243 100644
--- a/ews/coordinator/processor_scraper.py
+++ b/ews/coordinator/processor_scraper.py
@@ -465,4 +465,4 @@ class ProcessorScraper(ProcessorBase):
if __name__ == '__main__':
processor = ProcessorScraper()
- processor.run_processor("Scraper")
+ processor.run_processor("Scraper", "SCRAPER")
diff --git a/ews/coordinator/processor_surveys.py b/ews/coordinator/processor_surveys.py
index 30ab67b..dbc667a 100644
--- a/ews/coordinator/processor_surveys.py
+++ b/ews/coordinator/processor_surveys.py
@@ -424,4 +424,4 @@ class ProcessorSurveys(ProcessorBase):
if __name__ == '__main__':
print(__name__)
processor = ProcessorSurveys()
- processor.run_processor("Survey")
+ processor.run_processor("Survey", "SURVEYDATA")
diff --git a/ews/coordinator/utils/processor_utils.py b/ews/coordinator/utils/processor_utils.py
index 78f90a6..9b8248d 100644
--- a/ews/coordinator/utils/processor_utils.py
+++ b/ews/coordinator/utils/processor_utils.py
@@ -36,8 +36,7 @@ short_name = {
'Epidemiology' : 'EPI',
'Survey' : 'SURVEYDATA',
'Scraper' : 'SCRAPER',
- 'MetResample' : 'MET_RESAMPLE',
- 'MetPlotting' : 'MetPlotting',
+ 'MetResample' : 'MET_RESAMPLE'
}
disease_latin_name_dict = {
diff --git a/tests/integration/partial/integration_test_utils.py b/tests/integration/partial/integration_test_utils.py
index 9a02b0c..340298c 100644
--- a/tests/integration/partial/integration_test_utils.py
+++ b/tests/integration/partial/integration_test_utils.py
@@ -171,6 +171,7 @@ class IntegrationTestUtils:
@staticmethod
def run_partial_integration_test_pipeline(component: str,
+ shortname: str,
start_date: str,
processor: ProcessorBase,
**kwargs):
@@ -181,6 +182,7 @@ class IntegrationTestUtils:
:param processor:
:param component:
+ :param shortname:
:param start_date:
:param kwargs:
:return:
@@ -195,6 +197,7 @@ class IntegrationTestUtils:
args_dict['noupload'] = True
args_dict['start_date'] = start_date
args_dict['component'] = component
+ args_dict['short_name'] = shortname
args_dict['config_paths'] = config_paths
args_dict['log_level'] = 'info'
args_dict['clearup'] = True
diff --git a/tests/integration/partial/test_advisory.py b/tests/integration/partial/test_advisory.py
index 2ca6ef8..d72450e 100644
--- a/tests/integration/partial/test_advisory.py
+++ b/tests/integration/partial/test_advisory.py
@@ -69,10 +69,13 @@ class TestAdvisory(BaseAdvisoryTestSuite.AdvisoryTestSuite):
@staticmethod
def run_advisory_pipeline():
component = 'Advisory'
+ short_name = 'ADVISORY'
advisory_processor = ProcessorAdvisory()
IntegrationTestUtils.run_partial_integration_test_pipeline(component,
+ short_name,
IntegrationTestUtils.TEST_START_DATE,
advisory_processor)
+
if __name__ == '__main__':
unittest.main()
diff --git a/tests/integration/partial/test_deposition.py b/tests/integration/partial/test_deposition.py
index f8f214c..f4d9b20 100644
--- a/tests/integration/partial/test_deposition.py
+++ b/tests/integration/partial/test_deposition.py
@@ -64,10 +64,12 @@ class TestDeposition(BaseDepoTestSuite.DepoTestSuite):
@staticmethod
def run_depo_pipeline():
component = 'Deposition'
+ short_name = 'DEPOSITION'
# need EMAIL_CRED in the environment before we create a Processor
os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
deposition_processor = ProcessorDeposition()
IntegrationTestUtils.run_partial_integration_test_pipeline(component,
+ short_name,
IntegrationTestUtils.TEST_START_DATE,
deposition_processor)
diff --git a/tests/integration/partial/test_env_suit.py b/tests/integration/partial/test_env_suit.py
index 38bde21..9ade40d 100644
--- a/tests/integration/partial/test_env_suit.py
+++ b/tests/integration/partial/test_env_suit.py
@@ -67,10 +67,12 @@ class TestEnvSuit(BaseEnvSuitTestSuite.EnvSuitTestSuite):
@staticmethod
def run_env_pipeline():
component = 'Environment'
+ short_name = 'ENVIRONMENT_2.0'
# need EMAIL_CRED in the environment before we create a Processor
os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
environment_processor = ProcessorEnvironment()
IntegrationTestUtils.run_partial_integration_test_pipeline(component,
+ short_name,
IntegrationTestUtils.TEST_START_DATE,
environment_processor)
diff --git a/tests/integration/partial/test_epi.py b/tests/integration/partial/test_epi.py
index 6583761..b9466d6 100644
--- a/tests/integration/partial/test_epi.py
+++ b/tests/integration/partial/test_epi.py
@@ -67,10 +67,12 @@ class TestEpi(BaseEpiTestSuite.EpiTestSuite):
@staticmethod
def run_epi_pipeline():
component = 'Epidemiology'
+ short_name = 'EPI'
# need EMAIL_CRED in the environment before we create a Processor
os.environ["EMAIL_CRED"] = IntegrationTestUtils.EMAIL_CRED_PATH
epi_processor = ProcessorEpidemiology()
IntegrationTestUtils.run_partial_integration_test_pipeline(component,
+ short_name,
IntegrationTestUtils.TEST_START_DATE,
epi_processor)
diff --git a/tests/integration/partial/test_survey.py b/tests/integration/partial/test_survey.py
index 0d7c2f8..52edb0f 100644
--- a/tests/integration/partial/test_survey.py
+++ b/tests/integration/partial/test_survey.py
@@ -65,8 +65,10 @@ class TestSurvey(BaseSurveyTestSuite.SurveyTestSuite):
@staticmethod
def run_survey_pipeline():
component = 'Survey'
+ short_name = 'SURVEYDATA'
survey_processor = ProcessorSurveys()
IntegrationTestUtils.run_partial_integration_test_pipeline(component,
+ short_name,
IntegrationTestUtils.TEST_START_DATE,
survey_processor)
--
GitLab