Newer
Older
image: lb584/ews_coordinator:latest
# NOTE, MAKE SURE ANY PIPELINE-LEVEL VARIABLES ARE UNIQUE TO THIS PIPELINE
# WHEN CALLED FROM A MULTI-PIPELINE TRIGGER, VARIABLES WITH SHARED NAMES WILL BE OVERWRITTEN
TEST_WORKSPACE_DIR: "/storage/app/EWS_prod/regions/EastAfrica/test_workspace/"
value: "20241015"
description: "Full-fat tests will pull data from the deployment environment. For robustness, the default option is a
pre-selected date we know works. To override the default date, manually enter the key value pair using the format
in this example e.g.: CUSTOM_RUN_DATE 20230115 - NOTE THAT IF YOU SELECT A CUSTOM DATE BEFORE THE CURRENT
SEASON_START DATE, THE ADVISORY TEST WILL FAIL AS IT WILL NOT FIND A SURVEY WITHIN THE CURRENT SEASON"
- RUN_TIME=$(date '+%Y-%m-%d_%H%M%S')
- echo "run time - $RUN_TIME"
- PACKAGES_DIR="$CI_PROJECT_DIR/packages/"
- echo "project dir - " $CI_PROJECT_DIR
- echo "output dir - " $OUTPUT_DIR
- echo "packages dir - " $PACKAGES_DIR
- echo "full-fat run date type - " FULL_FAT_RUN_DATE_TYPE
- echo "custom full-fat run date - " $CUSTOM_RUN_DATE
- python3 -V # Print out python version for debugging
- mkdir -p $PACKAGES_DIR
# - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-source-generation.git" $PACKAGES_DIR/source_gen
# - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_plotting.git" $PACKAGES_DIR/plotting
# - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-epimodel.git" $PACKAGES_DIR/epimodel
# - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-advisory-builder.git" $PACKAGES_DIR/advisory_builder
# - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_environmental_suitability_v2.git" $PACKAGES_DIR/environmental_suitability
# - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing
# - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-met-processing.git" $PACKAGES_DIR/met_processing
- epimodel=$PACKAGES_DIR/epimodel
- advisory=$PACKAGES_DIR/advisory_builder
- met_processing=$PACKAGES_DIR/met_processing
- met_processor=$PACKAGES_DIR/environmental_suitability
- plotting=$PACKAGES_DIR/plotting
- post_processing=$PACKAGES_DIR/post_processing
- source_gen=$PACKAGES_DIR/source_gen
- coordinator_tests=$CI_PROJECT_DIR/tests
- export PYTHONPATH=$PYTHONPATH:$coordinator:$coordinator_tests
- ls $PACKAGES_DIR
- python3 -m coverage run run_test_suite.py $CI_PROJECT_DIR/test_reports
L. Bower
committed
#half_fat_epi_tests:
# when: always
# stage: half-fat
# extends: .before_script
#
# script:
# - cd $CI_PROJECT_DIR/tests/integration/partial/
# - python3 -m coverage run -m unittest test_epi.py
# - python3 -m coverage report
# - python3 -m coverage html -d $CI_PROJECT_DIR/coverage
# - ls
# artifacts:
# when: always
# paths:
# - $CI_PROJECT_DIR/coverage
# - $CI_PROJECT_DIR/test_reports
# expire_in: 10 days
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- when: manual
allow_failure: true
cd $CI_PROJECT_DIR/tests/integration/full/
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/depo_config_EastAfrica_fc_live.json
L. Bower
committed
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--custom_run_date $CUSTOM_RUN_DATE
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- when: manual
allow_failure: true
L. Bower
committed
EMAIL_CRED="/storage/app/EWS_prod/envs/credentials/Cred_gmail.json"
OUTPUT_DIR="/storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/outputs/"
LOGGING_CONFIG_FILE_PATH="/storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/data_proc/6.0_epi_pipeline/inputs/configs/epi_log_config.json"
L. Bower
committed
export LOGGING_CONFIG_FILE_PATH
mkdir $CI_PROJECT_DIR/fitting_outputs
L. Bower
committed
/storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/data_proc/6.0_epi_pipeline/inputs/configs/sys_config_epi_remote.json
/storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/data_proc/6.0_epi_pipeline/inputs/configs/run_config_epi_long_remote.json
L. Bower
committed
$OUTPUT_DIR
2016
python3 $PACKAGES_DIR/epimodel/ews/epi_model/epi_fit/params_metrics_file_compare.py
/storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/outputs/optimum_parameters_metrics.csv
/storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/data_proc/6.0_epi_pipeline/inputs/configs/example_optimum_parameters_metrics.csv
/storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/outputs/metric_comparison.txt
L. Bower
committed
L. Bower
committed
cp -rv $OUTPUT_DIR/*.csv $OUTPUT_DIR/*.json $OUTPUT_DIR/*.txt $OUTPUT_DIR/parameter_sweep_heatmaps $CI_PROJECT_DIR/fitting_outputs/
L. Bower
committed
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- when: manual
allow_failure: true
tags: ["ewas-production"]
script:
- >
cd $CI_PROJECT_DIR/tests/integration/full/
python3 full_test_env_suit.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/env_suit_config_EastAfrica_fc_live.json
L. Bower
committed
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
paths:
- $CI_PROJECT_DIR/test_reports
expire_in: 10 days
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
tags: ["ewas-production"]
script:
- >
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/survey_config_EastAfrica_fc_live.json
L. Bower
committed
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
paths:
- $CI_PROJECT_DIR/test_reports
expire_in: 10 days
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- when: manual
allow_failure: true
stage: full-fat
extends: .before_script
tags: ["ewas-production"]
script:
- >
cd $CI_PROJECT_DIR/tests/integration/full/
SURVEY_RUN_DATE=$(date -d "$CUSTOM_RUN_DATE -1 day" +%Y%m%d)
python3 full_test_survey.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/survey_config_EastAfrica_fc_live.json
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--test_report_dir $CI_PROJECT_DIR/test_reports
--custom_run_date $SURVEY_RUN_DATE
--custom_dir_prefix temp_advisory_$RUN_TIME
python3 full_test_deposition.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/depo_config_EastAfrica_fc_live.json
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--test_report_dir $CI_PROJECT_DIR/test_reports
--custom_run_date $CUSTOM_RUN_DATE
--custom_dir_prefix temp_advisory_$RUN_TIME
python3 full_test_env_suit.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/env_suit_config_EastAfrica_fc_live.json
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--custom_run_date $CUSTOM_RUN_DATE
python3 full_test_advisory.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/advisory_config_EastAfrica_fc_live.json
L. Bower
committed
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--custom_run_date $CUSTOM_RUN_DATE
paths:
- $CI_PROJECT_DIR/test_reports
expire_in: 10 days
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- when: manual
allow_failure: true
stage: full-fat
extends: .before_script
tags: ["ewas-production"]
script:
- >
cd $CI_PROJECT_DIR/tests/integration/full/
python3 full_test_deposition.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/depo_config_EastAfrica_fc_live.json
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--test_report_dir $CI_PROJECT_DIR/test_reports
--custom_run_date $CUSTOM_RUN_DATE
python3 full_test_env_suit.py
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/env_suit_config_EastAfrica_fc_live.json
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--custom_run_date $CUSTOM_RUN_DATE
--sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
--config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/epi_config_EastAfrica_fc_live.json
L. Bower
committed
--outdir ${TEST_WORKSPACE_DIR}/integration/full/
--email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
--custom_run_date $CUSTOM_RUN_DATE
paths:
- $CI_PROJECT_DIR/test_reports
expire_in: 10 days
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- when: manual
allow_failure: true
- export PATH=$PATH:/home/ewsmanager/.local/bin
- echo "Pages accessible through ${CI_PAGES_URL} / ${PAGES_PREFIX}"
- mkdir $CI_PROJECT_DIR/something
- mv -v $CI_PROJECT_DIR/docs/_build/html/* $CI_PROJECT_DIR/something/
- ls $CI_PROJECT_DIR/something/