From 768be24e79368fb6ffcbcf092013d7b9c9c7440e Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Wed, 1 Feb 2023 12:53:35 +0000
Subject: [PATCH] adding artifact for full fat test output
---
.gitlab-ci.yml | 40 ++++++++++++++++++++++++++--------------
1 file changed, 26 insertions(+), 14 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e6c33d6..9707842 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,8 +1,8 @@
-# Change pip's cache directory to be inside the project directory since we can
-# only cache local items.
+
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
PACKAGES_DIR: "$CI_PROJECT_DIR/packages/"
+ OUTPUT_DIR: "$CI_PROJECT_DIR/outputs/"
CI_PYTHON_PATH: "NOT_SET"
FULL_FAT_RUN_DATE:
value: "default"
@@ -17,11 +17,6 @@ variables:
CUSTOM_RUN_DATE: "NOT_SET"
-# Pip's cache doesn't store the python packages
-# https://pip.pypa.io/en/stable/reference/pip_install/#caching
-#
-# If you want to also cache the installed packages, you have to install
-# them in a virtualenv and cache it as well.
cache:
paths:
- .cache/pip
@@ -34,6 +29,7 @@ stages:
.half_fat_before_script:
before_script:
- python3 -V # Print out python version for debugging
+ - pip install coverage
- echo $PACKAGES_DIR
- mkdir $PACKAGES_DIR
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-source-generation.git" $PACKAGES_DIR/source_gen
@@ -44,9 +40,20 @@ stages:
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_environmental_suitability_v2.git" $PACKAGES_DIR/environmental_suitability
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_met_data_extraction.git" $PACKAGES_DIR/met_extractor_v2
+ - flagdir=$PACKAGES_DIR/flagdir
+ - epimodel=$PACKAGES_DIR/epimodel
+ - advisory=$PACKAGES_DIR/advisory_builder
+ - met_processing=$PACKAGES_DIR/met_extractor_v2/met_data_extraction
+ - met_processor=$PACKAGES_DIR/environmental_suitability/environmental_suitability
+ - plotting=$PACKAGES_DIR/plotting/plotting
+ - post_processing=$PACKAGES_DIR/post_processing/ews_postprocessing
+ - source_gen=$PACKAGES_DIR/source_gen
+ - coordinator=$CI_PROJECT_DIR/coordinator
+ - coordinator_tests=$CI_PROJECT_DIR/tests
+ - export PYTHONPATH=$PYTHONPATH:$coordinator:$coordinator_tests
+
- ls $PACKAGES_DIR
- source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/
- - pip install coverage
#.full_fat_before_script:
# before_script:
@@ -94,7 +101,7 @@ half_fat_tests:
artifacts:
paths:
- $CI_PROJECT_DIR/coverage
- expire_in: 30 days
+ expire_in: 10 days
half_fat_epi_tests:
when: manual
@@ -111,7 +118,7 @@ half_fat_epi_tests:
artifacts:
paths:
- $CI_PROJECT_DIR/coverage
- expire_in: 30 days
+ expire_in: 10 days
full_fat_tests:
when: manual
@@ -125,6 +132,8 @@ full_fat_tests:
echo $CUSTOM_RUN_DATE
+ echo $PYTHONPATH
+
echo "this is a manual job"
cd $CI_PROJECT_DIR/tests/integration/full/
@@ -135,8 +144,11 @@ full_fat_tests:
python3 full_test_deposition.py
--config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
- --outdir /storage/app/EWS_prod/regions/EastAfrica/workspace/
+ --outdir $OUTPUT_DIR
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
- --run_date_type $FULL_FAT_RUN_DATE
- --custom_run_date $CUSTOM_RUN_DATE
-
+ #--run_date_type $FULL_FAT_RUN_DATE
+ #--custom_run_date $CUSTOM_RUN_DATE
+ artifacts:
+ paths:
+ - $OUTPUT_DIR
+ expire_in: 10 days
--
GitLab