# Change pip's cache directory to be inside the project directory since we can
# only cache local items.
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
PACKAGES_DIR: "$CI_PROJECT_DIR/packages/"
CI_PYTHON_PATH: "NOT_SET"
DEPLOY_ENVIRONMENT:
value: "staging"
options:
- "production"
- "staging"
- "canary"
description: "The deployment target. Set to 'staging' by default."
# Pip's cache doesn't store the python packages
# https://pip.pypa.io/en/stable/reference/pip_install/#caching
#
# If you want to also cache the installed packages, you have to install
# them in a virtualenv and cache it as well.
cache:
paths:
- .cache/pip
- venv/
stages:
- half-fat
- full-fat
.half_fat_before_script:
before_script:
- python3 -V # Print out python version for debugging
- echo $PACKAGES_DIR
- mkdir $PACKAGES_DIR
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-source-generation.git" $PACKAGES_DIR/source_gen
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_plotting.git" $PACKAGES_DIR/plotting
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/flagdir.git" $PACKAGES_DIR/flagdir
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-epimodel.git" $PACKAGES_DIR/epimodel
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-advisory-builder.git" $PACKAGES_DIR/advisory_builder
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_environmental_suitability_v2.git" $PACKAGES_DIR/environmental_suitability
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing
- git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_met_data_extraction.git" $PACKAGES_DIR/met_extractor_v2
- ls $PACKAGES_DIR
- source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/
- pip install coverage
- pwd
# - flagdir=$PACKAGES_DIR/flagdir
# - epimodel=$PACKAGES_DIR/epimodel
# - advisory=$PACKAGES_DIR/advisory_builder
# - met_processing=$PACKAGES_DIR/met_extractor_v2/met_data_extraction
# - met_processor=$PACKAGES_DIR/environmental_suitability/environmental_suitability
# - plotting=$PACKAGES_DIR/plotting/plotting
# - post_processing=$PACKAGES_DIR/post_processing/ews_postprocessing
# - source_gen=$PACKAGES_DIR/source_gen
# - coordinator=$CI_PROJECT_DIR/coordinator
# - coordinator_tests=$CI_PROJECT_DIR/tests
# - export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests
- echo $PYTHONPATH
# - CI_PYTHON_PATH=$PYTHONPATH
# - ls ../
#.full_fat_before_script:
# before_script:
# - source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/
# - pip install coverage
# - pwd
# - CODE_DIR="/storage/app/EWS_prod/code/"
# - flagdir=$CODE_DIR/flagdir
# - epimodel=$CODE_DIR/epimodel
# - advisory=$CODE_DIR/advisory_builder
# - met_processing=$CODE_DIR/met_extractor_v2/met_data_extraction
# - met_processor=$CODE_DIR/environmental_suitability/environmental_suitability
# - plotting=$CODE_DIR/plotting/plotting
# - post_processing=$CODE_DIR/post_processing/ews_postprocessing
# - source_gen=$CODE_DIR/source_gen
# - coordinator=$CODE_DIR/coordinator/coordinator
# - coordinator_tests=$CODE_DIR/coordinator/tests
# - export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests
#
# - echo $PYTHONPATH
# - CI_PYTHON_PATH=$PYTHONPATH
half_fat_tests:
when: manual
stage: half-fat
extends: .half_fat_before_script
image: lb584/ews_coordinator:latest
script:
- export PYTHONPATH="$CI_PYTHON_PATH"
- echo $DEPLOY_ENVIRONMENT
# - echo $PYTHONPATH
- cd $CI_PROJECT_DIR/tests/unit/coordinator/
- python3 -m coverage run -m unittest *
- python -m coverage report
- python -m coverage html -d $CI_PROJECT_DIR/coverage
- cd $CI_PROJECT_DIR/tests/integration/partial/
- python3 -m coverage run -m unittest test_advisory.py
- python3 -m coverage run -m unittest test_deposition.py
- python3 -m coverage run -m unittest test_env_suit.py
- python3 -m coverage run -m unittest test_survey.py
- python3 -m coverage report
- python3 -m coverage html -d $CI_PROJECT_DIR/coverage
- ls
artifacts:
paths:
- $CI_PROJECT_DIR/coverage
expire_in: 30 days
half_fat_epi_tests:
when: manual
stage: half-fat
extends: .half_fat_before_script
image: lb584/ews_coordinator:latest
script:
- cd $CI_PROJECT_DIR/tests/integration/partial/
- python3 -m coverage run -m unittest test_epi.py
- python3 -m coverage report
- python3 -m coverage html -d $CI_PROJECT_DIR/coverage
- ls
artifacts:
paths:
- $CI_PROJECT_DIR/coverage
expire_in: 30 days
full_fat_tests:
stage: full-fat
extends: .half_fat_before_script
image: lb584/ews_coordinator:latest
tags: ["ewas-production"]
script:
# - export PYTHONPATH="$CI_PYTHON_PATH"
- echo $DEPLOY_ENVIRONMENT
# - echo $PYTHONPATH
- echo "this is a manual job"
- cd $CI_PROJECT_DIR/tests/integration/full/
- ls
# - rm /storage/app/EWS_prod/regions/EastAfrica/workspace/DEPOSITION_20230126/STATUS_*
- python3 full_test_deposition.py --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json --outdir /storage/app/EWS_prod/regions/EastAfrica/workspace/ --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json