.gitlab-ci.yml

image: lb584/ews_coordinator:latest

variables:
  # NOTE, MAKE SURE ANY PIPELINE-LEVEL VARIABLES ARE UNIQUE TO THIS PIPELINE
  # WHEN CALLED FROM A MULTI-PIPELINE TRIGGER, VARIABLES WITH SHARED NAMES WILL BE OVERWRITTEN

  OUTPUT_DIR: "$CI_PROJECT_DIR/outputs/"
  TEST_WORKSPACE_DIR: "/storage/app/EWS_prod/regions/EastAfrica/test_workspace/"
  FULL_FAT_RUN_DATE:
    value: "default"
    options:
      - "default"
      - "today"
      - "yesterday"
      - "custom"
    description: "Full-fat tests will pull data from the deployment environment. For robustness, the default option is a
    pre-selected date we know works. Select \"today\" or \"yesterday\" to test with recent live data. If \"custom\" is 
    selected, manually enter the key value pair using the format in this example: CUSTOM_RUN_DATE 20230115"

  CUSTOM_RUN_DATE: "NOT_SET"

cache:
  paths:
    - .cache/pip
    - venv/

stages:
  - docs
  - half-fat
  - full-fat

.before_script:
  before_script:

      - BRANCH="main"
      - echo "branch - " $BRANCH
      - PACKAGES_DIR="$CI_PROJECT_DIR/packages/"
      - echo "project dir - " $CI_PROJECT_DIR
      - echo "output dir - " $OUTPUT_DIR
      - echo "packages dir - " $PACKAGES_DIR
      - echo "pip dir - " $PIP_CACHE_DIR
      - echo "full-fat run date - " $FULL_FAT_RUN_DATE
      - echo "custom full-fat run date - " $CUSTOM_RUN_DATE

      - source /storage/app/EWS_prod/envs/venv/ews_prod/bin/activate
      - pip install coverage
      - python3 -V  # Print out python version for debugging
      - mkdir -p $PACKAGES_DIR
      - mkdir -p $OUTPUT_DIR
      - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-source-generation.git" $PACKAGES_DIR/source_gen
      - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_plotting.git" $PACKAGES_DIR/plotting
      - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-epimodel.git" $PACKAGES_DIR/epimodel
      - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-advisory-builder.git" $PACKAGES_DIR/advisory_builder
      - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_environmental_suitability_v2.git" $PACKAGES_DIR/environmental_suitability
      - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing
      - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-met-processing.git" $PACKAGES_DIR/met_processing
      - epimodel=$PACKAGES_DIR/epimodel
      - advisory=$PACKAGES_DIR/advisory_builder
      - met_processing=$PACKAGES_DIR/met_processing
      - met_processor=$PACKAGES_DIR/environmental_suitability
      - plotting=$PACKAGES_DIR/plotting
      - post_processing=$PACKAGES_DIR/post_processing
      - source_gen=$PACKAGES_DIR/source_gen
      - coordinator=$CI_PROJECT_DIR
      - coordinator_tests=$CI_PROJECT_DIR/tests
      - export PYTHONPATH=$PYTHONPATH:$met_processor_temp:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests
      - echo "pythonpath - " $PYTHONPATH

      - ls $PACKAGES_DIR

half_fat_tests:
  when: always
  stage: half-fat
  extends: .before_script

  script:
    - cd $CI_PROJECT_DIR/tests/integration/partial/
    - python3 -m coverage run run_test_suite.py $CI_PROJECT_DIR/test_reports
    - python3 -m coverage report
    - python3 -m coverage html -d $CI_PROJECT_DIR/coverage
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/coverage
      - $CI_PROJECT_DIR/test_reports
    expire_in: 10 days

half_fat_epi_tests:
  when: manual
  stage: half-fat
  extends: .before_script

  script:
    - cd $CI_PROJECT_DIR/tests/integration/partial/
    - python3 -m coverage run -m unittest test_epi.py
    - python3 -m coverage report
    - python3 -m coverage html -d $CI_PROJECT_DIR/coverage
    - ls
  artifacts:
    paths:
      - $CI_PROJECT_DIR/coverage
      - $CI_PROJECT_DIR/test_reports
    expire_in: 10 days

full_fat_depo:
  when: manual
  stage: full-fat
  extends: .before_script
  tags: ["ewas-production"]
  script:
    - >
      cd $CI_PROJECT_DIR/tests/integration/full/
      
      python3 full_test_deposition.py 
      --sys_config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/depo_config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --test_report_dir $CI_PROJECT_DIR/test_reports
      --run_date_type $FULL_FAT_RUN_DATE
      --custom_run_date $CUSTOM_RUN_DATE

full_fat_env_suit:
  when: manual
  stage: full-fat
  extends: .before_script
  tags: ["ewas-production"]
  script:
    - >
      cd $CI_PROJECT_DIR/tests/integration/full/

      python3 full_test_env_suit.py 
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --run_date_type $FULL_FAT_RUN_DATE
      --custom_run_date $CUSTOM_RUN_DATE

full_fat_survey:
  when: manual
  stage: full-fat
  extends: .before_script
  tags: ["ewas-production"]
  script:
    - >
      cd $CI_PROJECT_DIR/tests/integration/full/
      
      python3 full_test_survey.py 
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --run_date_type $FULL_FAT_RUN_DATE
      --custom_run_date $CUSTOM_RUN_DATE

full_fat_advisory:
  when: manual
  stage: full-fat
  extends: .before_script
  tags: ["ewas-production"]
  script:
    - >
      cd $CI_PROJECT_DIR/tests/integration/full/

      python3 full_test_advisory.py 
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --run_date_type $FULL_FAT_RUN_DATE
      --custom_run_date $CUSTOM_RUN_DATE

full_fat_epi:
  when: manual
  stage: full-fat
  extends: .before_script
  tags: ["ewas-production"]
  script:
    - >
      cd $CI_PROJECT_DIR/tests/integration/full/

      python3 full_test_epi.py 
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
      --outdir ${TEST_WORKSPACE_DIR}/integration/full/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json
      --run_date_type $FULL_FAT_RUN_DATE
      --custom_run_date $CUSTOM_RUN_DATE

pages:
  when: manual
  stage: docs
  extends: .before_script
  script:
    - export PATH=$PATH:/home/ewsmanager/.local/bin
    - pip install sphinx
    - pip install sphinx_rtd_theme
    - pip install sphinx_autodoc_typehints
    - pip install IPython
    - cd $CI_PROJECT_DIR/docs
    - make html
    - mkdir $CI_PROJECT_DIR/public
    - mv -v $CI_PROJECT_DIR/docs/_build/html/* $CI_PROJECT_DIR/public/
    - ls $CI_PROJECT_DIR/public/
  artifacts:
    paths:
      - $CI_PROJECT_DIR/public