.gitlab-ci.yml

# Change pip's cache directory to be inside the project directory since we can
# only cache local items.
variables:
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  PACKAGES_DIR: "$CI_PROJECT_DIR/packages/"
  CI_PYTHON_PATH: "NOT_SET"
  FULL_FAT_RUN_DATE:
    value: "today"
    options:
      - "today"
      - "yesterday"
      - "custom"
    description: "Full-fat tests will pull data from the deployment environment. As with the production pipeline, not 
    specifying a date will result in todays date being used. If \"custom\" is selected, manually enter the key value 
    pair using the format in this example: CUSTOM_RUN_DATE 20230115"

  CUSTOM_RUN_DATE: "NOT_SET"

# Pip's cache doesn't store the python packages
# https://pip.pypa.io/en/stable/reference/pip_install/#caching
#
# If you want to also cache the installed packages, you have to install
# them in a virtualenv and cache it as well.
cache:
  paths:
    - .cache/pip
    - venv/

stages:
  - half-fat
  - full-fat

.half_fat_before_script:
  before_script:
      - python3 -V  # Print out python version for debugging
      - echo $PACKAGES_DIR
      - mkdir $PACKAGES_DIR
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-source-generation.git" $PACKAGES_DIR/source_gen
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_plotting.git" $PACKAGES_DIR/plotting
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/flagdir.git" $PACKAGES_DIR/flagdir
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-epimodel.git" $PACKAGES_DIR/epimodel
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-advisory-builder.git" $PACKAGES_DIR/advisory_builder
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_environmental_suitability_v2.git" $PACKAGES_DIR/environmental_suitability
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_met_data_extraction.git" $PACKAGES_DIR/met_extractor_v2
      - ls $PACKAGES_DIR
      - source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/
      - pip install coverage

#.full_fat_before_script:
#  before_script:
#      - source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/
#      - pip install coverage
#      - pwd
#      - CODE_DIR="/storage/app/EWS_prod/code/"
#      - flagdir=$CODE_DIR/flagdir
#      - epimodel=$CODE_DIR/epimodel
#      - advisory=$CODE_DIR/advisory_builder
#      - met_processing=$CODE_DIR/met_extractor_v2/met_data_extraction
#      - met_processor=$CODE_DIR/environmental_suitability/environmental_suitability
#      - plotting=$CODE_DIR/plotting/plotting
#      - post_processing=$CODE_DIR/post_processing/ews_postprocessing
#      - source_gen=$CODE_DIR/source_gen
#      - coordinator=$CODE_DIR/coordinator/coordinator
#      - coordinator_tests=$CODE_DIR/coordinator/tests
#      - export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests
#
#      - echo $PYTHONPATH
#      - CI_PYTHON_PATH=$PYTHONPATH

half_fat_tests:
  when: manual
  stage: half-fat
  extends: .half_fat_before_script
  image: lb584/ews_coordinator:latest

  script:
    - export PYTHONPATH="$CI_PYTHON_PATH"
    - echo $FULL_FAT_RUN_DATE
#    - echo $PYTHONPATH
    - cd $CI_PROJECT_DIR/tests/unit/coordinator/
    - python3 -m coverage run -m unittest *
    - python -m coverage report
    - python -m coverage html -d $CI_PROJECT_DIR/coverage
    - cd $CI_PROJECT_DIR/tests/integration/partial/
    - python3 -m coverage run -m unittest test_advisory.py
    - python3 -m coverage run -m unittest test_deposition.py
    - python3 -m coverage run -m unittest test_env_suit.py
    - python3 -m coverage run -m unittest test_survey.py
    - python3 -m coverage report
    - python3 -m coverage html -d $CI_PROJECT_DIR/coverage
    - ls
  artifacts:
    paths:
      - $CI_PROJECT_DIR/coverage
    expire_in: 30 days

half_fat_epi_tests:
  when: manual
  stage: half-fat
  extends: .half_fat_before_script
  image: lb584/ews_coordinator:latest

  script:
    - cd $CI_PROJECT_DIR/tests/integration/partial/
    - python3 -m coverage run -m unittest test_epi.py
    - python3 -m coverage report
    - python3 -m coverage html -d $CI_PROJECT_DIR/coverage
    - ls
  artifacts:
    paths:
      - $CI_PROJECT_DIR/coverage
    expire_in: 30 days

full_fat_tests:
  when: manual
  stage: full-fat
  extends: .half_fat_before_script
  image: lb584/ews_coordinator:latest
  tags: ["ewas-production"]
  script:
    ->
      echo $FULL_FAT_RUN_DATE
    
      echo $CUSTOM_RUN_DATE
#    - echo $PYTHONPATH
    
      echo "this is a manual job"
    
      cd $CI_PROJECT_DIR/tests/integration/full/
    
      ls
#    - rm /storage/app/EWS_prod/regions/EastAfrica/workspace/DEPOSITION_20230126/STATUS_*
    
      python3 full_test_deposition.py
      --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
      --outdir /storage/app/EWS_prod/regions/EastAfrica/workspace/
      --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json