variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
PACKAGES_DIR: "$CI_PROJECT_DIR/packages/"
OUTPUT_DIR: "$CI_PROJECT_DIR/outputs/"
# OUTPUT_DIR: "/storage/app/EWS_prod/regions/EastAfrica/workspace/"
TEST_WORKSPACE_DIR: "/storage/app/EWS_prod/regions/EastAfrica/test_workspace/"
FULL_FAT_RUN_DATE:
value: "default"
options:
- "default"
- "today"
- "yesterday"
- "custom"
description: "Full-fat tests will pull data from the deployment environment. For robustness, the default option is a
pre-selected date we know works. Select \"today\" or \"yesterday\" to test with recent live data. If \"custom\" is
selected, manually enter the key value pair using the format in this example: CUSTOM_RUN_DATE 20230115"
CUSTOM_RUN_DATE: "NOT_SET"
FULL_FAT_COPY_OUTPUT:
value: "false"
options:
- "true"
- "false"
description: "Select \"true\" to make a copy of the full-fat integration test output. Outputs will be copied to the
production machine at /storage/app/EWS_prod/regions/EastAfrica/test_workspace"
cache:
paths:
- .cache/pip
- venv/
stages:
- half-fat
- full-fat
.half_fat_before_script:
before_script:
- echo "packages dir - " $PACKAGES_DIR
- echo "full-fat run date - " $FULL_FAT_RUN_DATE
- echo "custom full-fat run date - " $CUSTOM_RUN_DATE
- echo "output dir - " $OUTPUT_DIR
- source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/
- pip install coverage
- python3 -V # Print out python version for debugging
- mkdir $PACKAGES_DIR
- mkdir -p $OUTPUT_DIR
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-source-generation.git" $PACKAGES_DIR/source_gen
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_plotting.git" $PACKAGES_DIR/plotting
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/flagdir.git" $PACKAGES_DIR/flagdir
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-epimodel.git" $PACKAGES_DIR/epimodel
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-advisory-builder.git" $PACKAGES_DIR/advisory_builder
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_environmental_suitability_v2.git" $PACKAGES_DIR/environmental_suitability
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_met_data_extraction.git" $PACKAGES_DIR/met_extractor_v2
- flagdir=$PACKAGES_DIR/flagdir
- epimodel=$PACKAGES_DIR/epimodel
- advisory=$PACKAGES_DIR/advisory_builder
- met_processing=$PACKAGES_DIR/met_extractor_v2/met_data_extraction
- met_processor=$PACKAGES_DIR/environmental_suitability/environmental_suitability
- plotting=$PACKAGES_DIR/plotting/plotting
- post_processing=$PACKAGES_DIR/post_processing/ews_postprocessing
- source_gen=$PACKAGES_DIR/source_gen
- coordinator=$CI_PROJECT_DIR/coordinator
- coordinator_tests=$CI_PROJECT_DIR/tests
- export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests
- echo "pythonpath - " $PYTHONPATH
- ls $PACKAGES_DIR
#.full_fat_before_script:
# before_script:
# - source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/
# - pip install coverage
# - pwd
# - CODE_DIR="/storage/app/EWS_prod/code/"
# - flagdir=$CODE_DIR/flagdir
# - epimodel=$CODE_DIR/epimodel
# - advisory=$CODE_DIR/advisory_builder
# - met_processing=$CODE_DIR/met_extractor_v2/met_data_extraction
# - met_processor=$CODE_DIR/environmental_suitability/environmental_suitability
# - plotting=$CODE_DIR/plotting/plotting
# - post_processing=$CODE_DIR/post_processing/ews_postprocessing
# - source_gen=$CODE_DIR/source_gen
# - coordinator=$CODE_DIR/coordinator/coordinator
# - coordinator_tests=$CODE_DIR/coordinator/tests
# - export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests
#
# - echo $PYTHONPATH
half_fat_tests:
when: always
stage: half-fat
extends: .half_fat_before_script
image: lb584/ews_coordinator:latest
script:
- cd $CI_PROJECT_DIR/tests/integration/partial/
- python3 -m coverage run run_test_suite.py $CI_PROJECT_DIR/test_reports
- python3 -m coverage report
- python3 -m coverage html -d $CI_PROJECT_DIR/coverage
# - ls
artifacts:
when: always
paths:
- $CI_PROJECT_DIR/coverage
- $CI_PROJECT_DIR/test_reports
reports:
junit: report.xml
expire_in: 10 days
half_fat_epi_tests:
when: manual
stage: half-fat
extends: .half_fat_before_script
image: lb584/ews_coordinator:latest
script:
- cd $CI_PROJECT_DIR/tests/integration/partial/
- python3 -m coverage run -m unittest test_epi.py
- python3 -m coverage report
- python3 -m coverage html -d $CI_PROJECT_DIR/coverage
- ls
artifacts:
paths:
- $CI_PROJECT_DIR/coverage
- $CI_PROJECT_DIR/test_reports
expire_in: 10 days
full_fat_depo:
when: manual
stage: full-fat
extends: .half_fat_before_script
image: lb584/ews_coordinator:latest
tags: ["ewas-production"]
script:
- >
cd $CI_PROJECT_DIR/tests/integration/full/
python3 full_test_deposition.py
--config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
--outdir $OUTPUT_DIR
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--run_date_type $FULL_FAT_RUN_DATE
--custom_run_date $CUSTOM_RUN_DATE
after_script:
- >
if [[ "$FULL_FAT_COPY_OUTPUT" == "true" ]];
then
date=$(date '+%Y-%m-%d-%H%M');
mkdir ${TEST_WORKSPACE_DIR}/integration/full/${date};
cp -r $OUTPUT_DIR/* ${TEST_WORKSPACE_DIR}/integration/full/${date};
fi
artifacts:
when: on_failure
paths:
- $OUTPUT_DIR/*/log.txt
- $OUTPUT_DIR/*.txt
expire_in: 10 days