diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9627ee5cbfe027b2aaf3e26f393b438315aef5c7..215657283ecaa4cafa6eeaa653398eac94c56f36 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -138,21 +138,26 @@ epi_fit: export EMAIL_CRED + OUTPUT_DIR="/storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/outputs/" + python3 $PACKAGES_DIR/epimodel/ews/epi_model/fit_lb584.py /storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/data_proc/6.0_epi_pipeline/inputs/configs/sys_config_epi_remote.json /storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/data_proc/6.0_epi_pipeline/inputs/configs/run_config_epi_long_remote.json - /storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/outputs/ + $OUTPUT_DIR 2016 mkdir $CI_PROJECT_DIR/fitting_outputs - cp -rv /storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/outputs/preproc_fit/* $CI_PROJECT_DIR/fitting_outputs/ + cp -rv $OUTPUT_DIR/*.csv $CI_PROJECT_DIR/fitting_outputs/ + cp -rv $OUTPUT_DIR/*.json $CI_PROJECT_DIR/fitting_outputs/ + cp -rv $OUTPUT_DIR/parameter_sweep_heatmaps $CI_PROJECT_DIR/fitting_outputs/ artifacts: when: always paths: - $CI_PROJECT_DIR/fitting_outputs/ expire_in: 10 days + full_fat_env_suit: when: manual stage: full-fat diff --git a/ews/coordinator/processor_base.py b/ews/coordinator/processor_base.py index 241e617a3f30ca2e89732b062c7617632e0a8ed4..1ca3da04b6f0bf102fa72f8956188d643e317346 100755 --- a/ews/coordinator/processor_base.py +++ b/ews/coordinator/processor_base.py @@ -310,6 +310,8 @@ class ProcessorBase: the sys_config in the run_config. Additionally, values set in the args are used to supplement the config dict generated form the json. + Note we dont use the logger in this method as it is called before the logging is set up. + :param args: :param config_path: :param sys_config_path: @@ -329,8 +331,8 @@ class ProcessorBase: if k not in config.keys(): config[k] = v else: - logger.warning(f"Key {k} already present in run config - not adding key with same name from " - f"the sys_config") + print(f"Key {k} already present in run config - not adding key with same name from " + f"the sys_config") """ then add the args keys and values to the config, we will override any keys that are already @@ -338,8 +340,8 @@ class ProcessorBase: """ for k, v in args.items(): if k in config.keys(): - logger.warning(f"Key {k} already present in run config - overriding key with same name from " - f"the sys args") + print(f"Key {k} already present in run config - overriding key with same name from " + f"the sys args") config[k] = v # set various time values based on the start_date that is passed in on the args @@ -350,7 +352,7 @@ class ProcessorBase: config['StartTime'] = start_time_string except: - logger.exception(f"Failure in opening or checking config {config_path}") + print(f"Failure in opening or checking config {config_path}") # TODO: This case should test flagdir.jobStatus.__exit__() raise # endJob('ERROR',premature=True)