changing artifact assets

.gitlab-ci.yml

@@ -138,21 +138,26 @@ epi_fit:
 
       export EMAIL_CRED
       
+      OUTPUT_DIR="/storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/outputs/"
+      
       python3 $PACKAGES_DIR/epimodel/ews/epi_model/fit_lb584.py       
       /storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/data_proc/6.0_epi_pipeline/inputs/configs/sys_config_epi_remote.json 
       /storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/data_proc/6.0_epi_pipeline/inputs/configs/run_config_epi_long_remote.json 
-      /storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/outputs/
+      $OUTPUT_DIR
       2016
       
       mkdir $CI_PROJECT_DIR/fitting_outputs
       
-      cp -rv /storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/outputs/preproc_fit/* $CI_PROJECT_DIR/fitting_outputs/
+      cp -rv $OUTPUT_DIR/*.csv $CI_PROJECT_DIR/fitting_outputs/
+      cp -rv $OUTPUT_DIR/*.json $CI_PROJECT_DIR/fitting_outputs/
+      cp -rv $OUTPUT_DIR/parameter_sweep_heatmaps $CI_PROJECT_DIR/fitting_outputs/
   artifacts:
     when: always
     paths:
       - $CI_PROJECT_DIR/fitting_outputs/
     expire_in: 10 days
 
+
 full_fat_env_suit:
   when: manual
   stage: full-fat

ews/coordinator/processor_base.py

@@ -310,6 +310,8 @@ class ProcessorBase:
         the sys_config in the run_config. Additionally, values set in the args are used to supplement the config dict
         generated form the json.
 
+        Note we dont use the logger in this method as it is called before the logging is set up.
+
         :param args:
         :param config_path:
         :param sys_config_path:
@@ -329,8 +331,8 @@ class ProcessorBase:
                 if k not in config.keys():
                     config[k] = v
                 else:
-                    logger.warning(f"Key {k} already present in run config - not adding key with same name from "
-                                   f"the sys_config")
+                    print(f"Key {k} already present in run config - not adding key with same name from "
+                          f"the sys_config")
 
             """
             then add the args keys and values to the config, we will override any keys that are already 
@@ -338,8 +340,8 @@ class ProcessorBase:
             """
             for k, v in args.items():
                 if k in config.keys():
-                    logger.warning(f"Key {k} already present in run config - overriding key with same name from "
-                                   f"the sys args")
+                    print(f"Key {k} already present in run config - overriding key with same name from "
+                          f"the sys args")
                 config[k] = v
 
             #  set various time values based on the start_date that is passed in on the args
@@ -350,7 +352,7 @@ class ProcessorBase:
             config['StartTime'] = start_time_string
 
         except:
-            logger.exception(f"Failure in opening or checking config {config_path}")
+            print(f"Failure in opening or checking config {config_path}")
             # TODO: This case should test flagdir.jobStatus.__exit__()
             raise  # endJob('ERROR',premature=True)