.gitlab-ci.yml

image: lb584/ews_coordinator:latest

variables:
  # NOTE, MAKE SURE ANY PIPELINE-LEVEL VARIABLES ARE UNIQUE TO THIS PIPELINE
  # WHEN CALLED FROM A MULTI-PIPELINE TRIGGER, VARIABLES WITH SHARED NAMES WILL BE OVERWRITTEN

  OUTPUT_DIR: "$CI_PROJECT_DIR/outputs/"
  TEST_WORKSPACE_DIR: "/storage/app/EWS_prod/regions/EastAfrica/test_workspace/"

  CUSTOM_RUN_DATE:
    value: "20240715"
    description: "Full-fat tests will pull data from the deployment environment. For robustness, the default option is a
    pre-selected date we know works. To override the default date, manually enter the key value pair using the format 
    in this example e.g.: CUSTOM_RUN_DATE 20230115 - NOTE THAT IF YOU SELECT A CUSTOM DATE BEFORE THE CURRENT 
    SEASON_START DATE, THE ADVISORY TEST WILL FAIL AS IT WILL NOT FIND A SURVEY WITHIN THE CURRENT SEASON"

cache:
  paths:
    - .cache/pip
    - venv/

stages:
  - docs
  - half-fat
  - full-fat
  - epi-fit

.before_script:
  before_script:

      - BRANCH="main"
      - echo "branch - " $BRANCH
      - RUN_TIME=$(date '+%Y-%m-%d_%H%M%S')
      - echo "run time - $RUN_TIME"
      - PACKAGES_DIR="$CI_PROJECT_DIR/packages/"
      - echo "project dir - " $CI_PROJECT_DIR
      - echo "output dir - " $OUTPUT_DIR
      - echo "packages dir - " $PACKAGES_DIR
      - echo "pip dir - " $PIP_CACHE_DIR
      - echo "full-fat run date type - " FULL_FAT_RUN_DATE_TYPE
      - echo "custom full-fat run date - " $CUSTOM_RUN_DATE

      - source /storage/app/EWS_prod/envs/venv/ews_prod/bin/activate
      - pip install coverage
      - python3 -V  # Print out python version for debugging
      - mkdir -p $PACKAGES_DIR
      - mkdir -p $OUTPUT_DIR
      - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-source-generation.git" $PACKAGES_DIR/source_gen
      - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_plotting.git" $PACKAGES_DIR/plotting
      - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-epimodel.git" $PACKAGES_DIR/epimodel
      - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-advisory-builder.git" $PACKAGES_DIR/advisory_builder
      - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_environmental_suitability_v2.git" $PACKAGES_DIR/environmental_suitability
      - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing
      - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-met-processing.git" $PACKAGES_DIR/met_processing
      - epimodel=$PACKAGES_DIR/epimodel
      - advisory=$PACKAGES_DIR/advisory_builder
      - met_processing=$PACKAGES_DIR/met_processing
      - met_processor=$PACKAGES_DIR/environmental_suitability
      - plotting=$PACKAGES_DIR/plotting
      - post_processing=$PACKAGES_DIR/post_processing
      - source_gen=$PACKAGES_DIR/source_gen
      - coordinator=$CI_PROJECT_DIR
      - coordinator_tests=$CI_PROJECT_DIR/tests
      - export PYTHONPATH=$PYTHONPATH:$met_processor_temp:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests
      - echo "pythonpath - " $PYTHONPATH

      - ls $PACKAGES_DIR

half_fat_tests:
  when: manual
  stage: half-fat
  extends: .before_script

  script:
    - cd $CI_PROJECT_DIR/tests/integration/partial/
    - python3 -m coverage run run_test_suite.py $CI_PROJECT_DIR/test_reports
    - python3 -m coverage report
    - python3 -m coverage html -d $CI_PROJECT_DIR/coverage
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/coverage
      - $CI_PROJECT_DIR/test_reports
    expire_in: 10 days

half_fat_epi_tests:
  when: manual
  stage: half-fat
  extends: .before_script

  script:
    - cd $CI_PROJECT_DIR/tests/integration/partial/
    - python3 -m coverage run -m unittest test_epi.py
    - python3 -m coverage report
    - python3 -m coverage html -d $CI_PROJECT_DIR/coverage
    - ls
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/coverage
      - $CI_PROJECT_DIR/test_reports
    expire_in: 10 days

full_fat_depo:
  when: manual
  stage: full-fat
  extends: .before_script
  tags: ["ewas-production"]
  script:
    - >
      cd $CI_PROJECT_DIR/tests/integration/full/
      
      python3 full_test_deposition.py 
      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/depo_config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --test_report_dir $CI_PROJECT_DIR/test_reports
      --custom_run_date $CUSTOM_RUN_DATE
      --custom_dir_prefix temp_depo_$RUN_TIME
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/test_reports
    expire_in: 10 days

epi_fit:
  when: manual
  stage: epi-fit
  extends: .before_script
  tags: ["ewas-production"]
  script:
    - >
      cd $CI_PROJECT_DIR
      mkdir $CI_PROJECT_DIR/outputs
      python3 $PACKAGES_DIR/epimodel/ews/epi_model/fit.py 
      /storage/app/EWS_prod/regions/EastAfrica/test_workspace/integration/epi_fit/dev/data_proc/6.0_epi_pipeline/inputs/configs/epi_fit_S3_H4_D1_M1_F2_Q1_short_epi.json 
      $CI_PROJECT_DIR/outputs/ 
      2016
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/outputs/
    expire_in: 10 days

full_fat_env_suit:
  when: manual
  stage: full-fat
  extends: .before_script
  tags: ["ewas-production"]
  script:
    - >
      cd $CI_PROJECT_DIR/tests/integration/full/

      python3 full_test_env_suit.py 
      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/env_suit_config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --test_report_dir $CI_PROJECT_DIR/test_reports
      --custom_run_date $CUSTOM_RUN_DATE
      --custom_dir_prefix temp_env_suit_$RUN_TIME
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/test_reports
    expire_in: 10 days

full_fat_survey:
  when: manual
  stage: full-fat
  extends: .before_script
  tags: ["ewas-production"]
  script:
    - >
      cd $CI_PROJECT_DIR/tests/integration/full/
      
      python3 full_test_survey.py 
      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/survey_config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --test_report_dir $CI_PROJECT_DIR/test_reports
      --custom_run_date $CUSTOM_RUN_DATE
      --custom_dir_prefix temp_survey_$RUN_TIME
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/test_reports
    expire_in: 10 days


full_fat_advisory:
  when: manual
  stage: full-fat
  extends: .before_script
  tags: ["ewas-production"]
  script:
    - >
      cd $CI_PROJECT_DIR/tests/integration/full/

      SURVEY_RUN_DATE=$(date -d "$CUSTOM_RUN_DATE -1 day" +%Y%m%d)

      python3 full_test_survey.py
      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/survey_config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --test_report_dir $CI_PROJECT_DIR/test_reports
      --custom_run_date $SURVEY_RUN_DATE
      --custom_dir_prefix temp_advisory_$RUN_TIME

      python3 full_test_deposition.py
      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/depo_config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --test_report_dir $CI_PROJECT_DIR/test_reports
      --custom_run_date $CUSTOM_RUN_DATE
      --custom_dir_prefix temp_advisory_$RUN_TIME

      python3 full_test_env_suit.py 
      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/env_suit_config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --test_report_dir $CI_PROJECT_DIR/test_reports
      --custom_run_date $CUSTOM_RUN_DATE
      --custom_dir_prefix temp_advisory_$RUN_TIME
      
      python3 full_test_advisory.py 
      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/advisory_config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --test_report_dir $CI_PROJECT_DIR/test_reports
      --custom_run_date $CUSTOM_RUN_DATE
      --custom_dir_prefix temp_advisory_$RUN_TIME
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/test_reports
    expire_in: 10 days

full_fat_epi:
  when: manual
  stage: full-fat
  extends: .before_script
  tags: ["ewas-production"]
  script:
    - >
      cd $CI_PROJECT_DIR/tests/integration/full/

      python3 full_test_deposition.py 
      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/depo_config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --test_report_dir $CI_PROJECT_DIR/test_reports
      --custom_run_date $CUSTOM_RUN_DATE
      --custom_dir_prefix temp_epi_$RUN_TIME

      python3 full_test_env_suit.py 
      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/env_suit_config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --test_report_dir $CI_PROJECT_DIR/test_reports
      --custom_run_date $CUSTOM_RUN_DATE
      --custom_dir_prefix temp_epi_$RUN_TIME

      python3 full_test_epi.py 
      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/epi_config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --test_report_dir $CI_PROJECT_DIR/test_reports
      --custom_run_date $CUSTOM_RUN_DATE
      --custom_dir_prefix temp_epi_$RUN_TIME
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/test_reports
    expire_in: 10 days

pages:
  when: manual
  stage: docs
  extends: .before_script
  script:
    - export PATH=$PATH:/home/ewsmanager/.local/bin
    - pip install sphinx
    - pip install sphinx_rtd_theme
    - pip install sphinx_autodoc_typehints
    - pip install IPython
    - cd $CI_PROJECT_DIR/docs
    - make html
    - mkdir $CI_PROJECT_DIR/public
    - mv -v $CI_PROJECT_DIR/docs/_build/html/* $CI_PROJECT_DIR/public/
    - ls $CI_PROJECT_DIR/public/
  artifacts:
    paths:
      - $CI_PROJECT_DIR/public