From 96c558d02f520d3a1e9ba618ab2683d8dac14ff9 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Thu, 5 Sep 2024 12:09:22 +0100
Subject: [PATCH] changing artifact assets
---
.gitlab-ci.yml | 9 +++++++--
ews/coordinator/processor_base.py | 12 +++++++-----
2 files changed, 14 insertions(+), 7 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9627ee5..2156572 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -138,21 +138,26 @@ epi_fit:
export EMAIL_CRED
+ OUTPUT_DIR="/storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/outputs/"
+
python3 $PACKAGES_DIR/epimodel/ews/epi_model/fit_lb584.py
/storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/data_proc/6.0_epi_pipeline/inputs/configs/sys_config_epi_remote.json
/storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/data_proc/6.0_epi_pipeline/inputs/configs/run_config_epi_long_remote.json
- /storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/outputs/
+ $OUTPUT_DIR
2016
mkdir $CI_PROJECT_DIR/fitting_outputs
- cp -rv /storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/outputs/preproc_fit/* $CI_PROJECT_DIR/fitting_outputs/
+ cp -rv $OUTPUT_DIR/*.csv $CI_PROJECT_DIR/fitting_outputs/
+ cp -rv $OUTPUT_DIR/*.json $CI_PROJECT_DIR/fitting_outputs/
+ cp -rv $OUTPUT_DIR/parameter_sweep_heatmaps $CI_PROJECT_DIR/fitting_outputs/
artifacts:
when: always
paths:
- $CI_PROJECT_DIR/fitting_outputs/
expire_in: 10 days
+
full_fat_env_suit:
when: manual
stage: full-fat
diff --git a/ews/coordinator/processor_base.py b/ews/coordinator/processor_base.py
index 241e617..1ca3da0 100755
--- a/ews/coordinator/processor_base.py
+++ b/ews/coordinator/processor_base.py
@@ -310,6 +310,8 @@ class ProcessorBase:
the sys_config in the run_config. Additionally, values set in the args are used to supplement the config dict
generated form the json.
+ Note we dont use the logger in this method as it is called before the logging is set up.
+
:param args:
:param config_path:
:param sys_config_path:
@@ -329,8 +331,8 @@ class ProcessorBase:
if k not in config.keys():
config[k] = v
else:
- logger.warning(f"Key {k} already present in run config - not adding key with same name from "
- f"the sys_config")
+ print(f"Key {k} already present in run config - not adding key with same name from "
+ f"the sys_config")
"""
then add the args keys and values to the config, we will override any keys that are already
@@ -338,8 +340,8 @@ class ProcessorBase:
"""
for k, v in args.items():
if k in config.keys():
- logger.warning(f"Key {k} already present in run config - overriding key with same name from "
- f"the sys args")
+ print(f"Key {k} already present in run config - overriding key with same name from "
+ f"the sys args")
config[k] = v
# set various time values based on the start_date that is passed in on the args
@@ -350,7 +352,7 @@ class ProcessorBase:
config['StartTime'] = start_time_string
except:
- logger.exception(f"Failure in opening or checking config {config_path}")
+ print(f"Failure in opening or checking config {config_path}")
# TODO: This case should test flagdir.jobStatus.__exit__()
raise # endJob('ERROR',premature=True)
--
GitLab