diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6755d1fd35be44326ddf0bac1016e92327baf806..feb11061abad026daee487fbea4676b6f96ed355 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,13 +50,11 @@ stages: - mkdir -p $OUTPUT_DIR - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-source-generation.git" $PACKAGES_DIR/source_gen - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_plotting.git" $PACKAGES_DIR/plotting - - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/flagdir.git" $PACKAGES_DIR/flagdir - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-epimodel.git" $PACKAGES_DIR/epimodel - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-advisory-builder.git" $PACKAGES_DIR/advisory_builder - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_environmental_suitability_v2.git" $PACKAGES_DIR/environmental_suitability - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-met-processing.git" $PACKAGES_DIR/met_processing - - flagdir=$PACKAGES_DIR/flagdir - epimodel=$PACKAGES_DIR/epimodel - advisory=$PACKAGES_DIR/advisory_builder - met_processing=$PACKAGES_DIR/met_processing @@ -66,7 +64,7 @@ stages: - source_gen=$PACKAGES_DIR/source_gen - coordinator=$CI_PROJECT_DIR - coordinator_tests=$CI_PROJECT_DIR/tests - - export PYTHONPATH=$PYTHONPATH:$met_processor_temp:$flagdir:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests + - export PYTHONPATH=$PYTHONPATH:$met_processor_temp:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests - echo "pythonpath - " $PYTHONPATH - ls $PACKAGES_DIR diff --git a/ews/coordinator/processor_base.py b/ews/coordinator/processor_base.py index 08985e86012f4f0130f8530aaf64008ab92fe5a8..5312047000c0215ce2a2da7c2ee7ba1f3b8b9e18 100755 --- a/ews/coordinator/processor_base.py +++ b/ews/coordinator/processor_base.py @@ -20,7 +20,7 @@ from abc import abstractmethod, ABCMeta from typing import List from ews.coordinator.utils import processor_utils -from ews.coordinator.utils.jobstatus import jobStatus +from ews.coordinator.utils.jobstatus import Jobstatus from ews.coordinator.utils.processor_utils import short_name, open_and_check_config, end_script, end_job, append_item_to_list, \ clear_up @@ -260,8 +260,8 @@ class ProcessorBase: ready = self.process_pre_job(args) # lock job directory - status: jobStatus - with jobStatus(job_path) as status: + status: Jobstatus + with Jobstatus(job_path) as status: # check for a status file in job directory if status.had_initial_status: diff --git a/ews/coordinator/utils/jobstatus.py b/ews/coordinator/utils/jobstatus.py index 23fb5421698bddf99f4d3418a4a25377b54d3bd6..740cd427374062c3e3951342fd3c5e924fb2cfc0 100644 --- a/ews/coordinator/utils/jobstatus.py +++ b/ews/coordinator/utils/jobstatus.py @@ -7,8 +7,8 @@ Running with `conda activate py3EWSepi`. import os from glob import glob -class jobStatus: - ''' +class Jobstatus: + """ Context manager to determine and change status of a job based on status file in a given directory. @@ -19,7 +19,7 @@ class jobStatus: Use jobStatus.get() to perform a fresh search for status file. - If you don't want any status files for this directory anymore, use + If you don't want any status files for this directory anymore, use jobStatus.cleanup() Suggested usage (in between task commands): @@ -36,7 +36,7 @@ class jobStatus: status = js.get() js.reset('SUCCESS') #js.cleanup() # if success file not needed - ''' + """ _status_dict = { #-1: None, @@ -53,15 +53,15 @@ class jobStatus: _default_end_status = 'ERROR' - def __init__(self,jobPath): + def __init__(self, job_path: str): # determine job directory - self.jobPath = jobPath + self.job_path = job_path - print(f"job path is {self.jobPath}") + print(f"job path is {self.job_path}") - if not os.path.isdir(self.jobPath): + if not os.path.isdir(self.job_path): raise FileNotFoundError def __enter__(self): @@ -98,11 +98,11 @@ class jobStatus: def cleanup(self): # remove any current status file - statusPaths = glob(f"{self.jobPath}/STATUS_*") + status_paths = glob(f"{self.job_path}/STATUS_*") #print(f"statusPaths are\n:{statusPaths}") - for f in statusPaths: + for f in status_paths: try: print(f' removing {f}') @@ -114,7 +114,8 @@ class jobStatus: return - def reset(self,new_status_in): + + def reset(self, new_status_in): # ensure status is acceptable # for now, handle string only @@ -122,7 +123,7 @@ class jobStatus: assert new_status_in in self._status_dict.values() # check statusPath has already been defined - assert hasattr(self,'statusPath') + assert hasattr(self,'status_path') # modify new status if a warning or error once occurred new_status = new_status_in @@ -130,17 +131,17 @@ class jobStatus: if (self.status in top_statuses) & (new_status not in top_statuses): new_status = f"{new_status_in}_WITH_{self.status}" - new_statusPath = f"{self.jobPath}/STATUS_{new_status}" + new_status_path = f"{self.job_path}/STATUS_{new_status}" - if self.statusPath is not None: + if self.status_path is not None: - print(f'Renaming {self.statusPath} to {new_statusPath}') + print(f'Renaming {self.status_path} to {new_status_path}') - os.rename(self.statusPath,new_statusPath) + os.rename(self.status_path, new_status_path) else: - print(f'Creating {new_statusPath}') + print(f'Creating {new_status_path}') # by running set below @@ -148,7 +149,8 @@ class jobStatus: return - def set(self,status): + + def set(self, status): # ensure it is acceptable status # for now, handle string only @@ -157,16 +159,17 @@ class jobStatus: assert status in self._status_dict.values() # set file status - self.statusPath = f"{self.jobPath}/STATUS_{status}" + self.status_path = f"{self.job_path}/STATUS_{status}" - open(self.statusPath,'a').close() + open(self.status_path, 'a').close() # set object status self.status = status return - def get_from_file_path(self,statusPath): - '''Convert path to status string.''' + + def get_from_file_path(self, statusPath): + """Convert path to status string.""" filename = statusPath.split('/')[-1] @@ -180,37 +183,39 @@ class jobStatus: return statusString def get_status_path(self): - '''This might not be needed all the time.''' + """This might not be needed all the time.""" - statusFiles = glob(f"{self.jobPath}/STATUS_*") + status_files = glob(f"{self.job_path}/STATUS_*") - if len(statusFiles) > 1: + if len(status_files) > 1: print('ERROR: require only one status file in job directory') raise AssertionError - elif len(statusFiles) == 0: + elif len(status_files) == 0: return None else: - return statusFiles[0] + return status_files[0] + + + def get(self, statuspath = None): - def get(self,statuspath=None): - '''Fresh look for status file and set object status based on that. - Does not modify self.status directly''' + """Fresh look for status file and set object status based on that. + Does not modify self.status directly""" if statuspath is None: - self.statusPath = self.get_status_path() + self.status_path = self.get_status_path() - if self.statusPath is None: + if self.status_path is None: status = None else: - status = self.get_from_file_path(self.statusPath) + status = self.get_from_file_path(self.status_path) return status # logical form of status def has_status(self): - return (hasattr(self,'status')) & (self.status in self._status_dict.values()) + return (hasattr(self, 'status')) & (self.status in self._status_dict.values()) def is_success(self): self.get() diff --git a/ews/coordinator/utils/processor_utils.py b/ews/coordinator/utils/processor_utils.py index de5267f8d1710e6536cde0e3671538ed75c212e2..18e489fe2c9614d52ed3152ba7a022440e9ae3d8 100644 --- a/ews/coordinator/utils/processor_utils.py +++ b/ews/coordinator/utils/processor_utils.py @@ -18,7 +18,7 @@ from typing import List from iris import load from iris.cube import CubeList -from ews.coordinator.utils.jobstatus import jobStatus +from ews.coordinator.utils.jobstatus import Jobstatus logger = logging.getLogger(__name__) @@ -178,7 +178,7 @@ def append_item_to_list( item: list, li: list, description: str, - status: jobStatus) -> None: + status: Jobstatus) -> None: if isinstance(item,list): li += item diff --git a/tests/unit/coordinator/utils/test_flagdir.py b/tests/unit/coordinator/utils/test_flagdir.py index ea793c1c42ace3128200b0fd1f1a603f185b3ef7..cf2fc8bb13f8d1812b047774177a7d87bbe3bffe 100644 --- a/tests/unit/coordinator/utils/test_flagdir.py +++ b/tests/unit/coordinator/utils/test_flagdir.py @@ -17,9 +17,9 @@ import os from ews.coordinator.utils import jobstatus -class Test_jobStatus(unittest.TestCase): - '''Functions to test file and directory manipulation.''' +class TestJobstatus(unittest.TestCase): + """Functions to test file and directory manipulation.""" # TODO: replace this SetUp and tearDown approach with a context manager, for more robust workspace deletion def setUp(self): @@ -39,37 +39,37 @@ class Test_jobStatus(unittest.TestCase): def test_context_call_assigns_status(self): - with jobstatus.jobStatus(self.jobdir) as js: + with jobstatus.Jobstatus(self.jobdir) as js: self.assertTrue(hasattr(js, 'status')) def test_direct_call_does_not_assign_status(self): # because it should be used as a context manager - self.assertFalse(hasattr(jobstatus.jobStatus(self.jobdir), 'status')) + self.assertFalse(hasattr(jobstatus.Jobstatus(self.jobdir), 'status')) def test_init_fails_when_path_not_exists(self): - self.assertRaises(FileNotFoundError, jobstatus.jobStatus, self.jobdir + 'fake') + self.assertRaises(FileNotFoundError, jobstatus.Jobstatus, self.jobdir + 'fake') def test_get_from_file_path_fails_if_bad_file_status(self): # make a status file in job directory - statusPath = self.jobdir + '/STATUS_UNKNOWN' - with open(statusPath, 'w'): pass + status_path = self.jobdir + '/STATUS_UNKNOWN' + with open(status_path, 'w'): pass - self.assertRaises(KeyError, jobstatus.jobStatus.get_from_file_path, jobstatus.jobStatus, statusPath) + self.assertRaises(KeyError, jobstatus.Jobstatus.get_from_file_path, jobstatus.Jobstatus, status_path) def test_set_can_create_status_file(self): - js = jobstatus.jobStatus(self.jobdir) + js = jobstatus.Jobstatus(self.jobdir) - statusPathExpected = self.jobdir + '/STATUS_SUCCESS' - self.assertFalse(os.path.isfile(statusPathExpected)) + status_path_expected = self.jobdir + '/STATUS_SUCCESS' + self.assertFalse(os.path.isfile(status_path_expected)) js.set('SUCCESS') - self.assertTrue(os.path.isfile(statusPathExpected)) + self.assertTrue(os.path.isfile(status_path_expected)) def test_get_gets_status_if_file_is_present(self): @@ -77,59 +77,59 @@ class Test_jobStatus(unittest.TestCase): statusPath = self.jobdir + '/STATUS_SUCCESS' with open(statusPath, 'w'): pass - js = jobstatus.jobStatus(self.jobdir) + js = jobstatus.Jobstatus(self.jobdir) self.assertIsInstance(js.get(), str) def test_cleanup_can_remove_status(self): # make a status file in job directory - statusPathStart = self.jobdir + '/STATUS_INPROGRESS' - with open(statusPathStart, 'w'): pass - self.assertTrue(os.path.isfile(statusPathStart)) + status_path_start = self.jobdir + '/STATUS_INPROGRESS' + with open(status_path_start, 'w'): pass + self.assertTrue(os.path.isfile(status_path_start)) - with jobstatus.jobStatus(self.jobdir) as js: + with jobstatus.Jobstatus(self.jobdir) as js: js.cleanup() - self.assertFalse(os.path.isfile(statusPathStart)) + self.assertFalse(os.path.isfile(status_path_start)) def test_reset_can_change_status(self): # make a status file in job directory - statusPathStart = self.jobdir + '/STATUS_INPROGRESS' - with open(statusPathStart, 'w'): pass + status_path_start = self.jobdir + '/STATUS_INPROGRESS' + with open(status_path_start, 'w'): pass - with jobstatus.jobStatus(self.jobdir) as js: + with jobstatus.Jobstatus(self.jobdir) as js: statusPathEnd = self.jobdir + '/STATUS_SUCCESS' js.reset('SUCCESS') - self.assertFalse(os.path.isfile(statusPathStart)) + self.assertFalse(os.path.isfile(status_path_start)) self.assertTrue(os.path.isfile(statusPathEnd)) def test_existing_status_is_untouched(self): # make a status file in job directory - statusPathStart = self.jobdir + '/STATUS_WARNING' - with open(statusPathStart, 'w'): pass + status_path_start = self.jobdir + '/STATUS_WARNING' + with open(status_path_start, 'w'): pass - with jobstatus.jobStatus(self.jobdir) as js: + with jobstatus.Jobstatus(self.jobdir) as js: pass - self.assertTrue(os.path.isfile(statusPathStart)) + self.assertTrue(os.path.isfile(status_path_start)) def test_existing_warning_status_is_remembered(self): # make a status file in job directory - statusPathStart = self.jobdir + '/STATUS_WARNING' - with open(statusPathStart, 'w'): pass + status_path_start = self.jobdir + '/STATUS_WARNING' + with open(status_path_start, 'w'): pass - with jobstatus.jobStatus(self.jobdir) as js: + with jobstatus.Jobstatus(self.jobdir) as js: js.reset('SUCCESS') - statusPathExpected = self.jobdir + '/STATUS_SUCCESS_WITH_WARNING' + status_path_expected = self.jobdir + '/STATUS_SUCCESS_WITH_WARNING' - self.assertTrue(os.path.isfile(statusPathExpected)) + self.assertTrue(os.path.isfile(status_path_expected)) def test_existing_error_status_is_remembered(self): @@ -137,7 +137,7 @@ class Test_jobStatus(unittest.TestCase): statusPathStart = self.jobdir + '/STATUS_ERROR' with open(statusPathStart, 'w'): pass - with jobstatus.jobStatus(self.jobdir) as js: + with jobstatus.Jobstatus(self.jobdir) as js: js.reset('SUCCESS') statusPathExpected = self.jobdir + '/STATUS_SUCCESS_WITH_ERROR'