.gitlab-ci.yml

# Change pip's cache directory to be inside the project directory since we can
# only cache local items.
variables:
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  PACKAGES_DIR: "$CI_PROJECT_DIR/packages/"
  CI_PYTHON_PATH: "NOT_SET"
  DEPLOY_ENVIRONMENT:
    value: "staging"
    options:
      - "production"
      - "staging"
      - "canary"
    description: "The deployment target. Set to 'staging' by default."

# Pip's cache doesn't store the python packages
# https://pip.pypa.io/en/stable/reference/pip_install/#caching
#
# If you want to also cache the installed packages, you have to install
# them in a virtualenv and cache it as well.
cache:
  paths:
    - .cache/pip
    - venv/

stages:
  - half-fat
  - full-fat

.half_fat_before_script:
  before_script:
      - python3 -V  # Print out python version for debugging
      - echo $PACKAGES_DIR
      - mkdir $PACKAGES_DIR
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-source-generation.git" $PACKAGES_DIR/source_gen
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_plotting.git" $PACKAGES_DIR/plotting
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/flagdir.git" $PACKAGES_DIR/flagdir
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-epimodel.git" $PACKAGES_DIR/epimodel
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-advisory-builder.git" $PACKAGES_DIR/advisory_builder
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_environmental_suitability_v2.git" $PACKAGES_DIR/environmental_suitability
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing
      - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_met_data_extraction.git" $PACKAGES_DIR/met_extractor_v2
      - ls $PACKAGES_DIR
      - source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/
      - pip install coverage
      - pwd
#      - flagdir=$PACKAGES_DIR/flagdir
#      - epimodel=$PACKAGES_DIR/epimodel
#      - advisory=$PACKAGES_DIR/advisory_builder
#      - met_processing=$PACKAGES_DIR/met_extractor_v2/met_data_extraction
#      - met_processor=$PACKAGES_DIR/environmental_suitability/environmental_suitability
#      - plotting=$PACKAGES_DIR/plotting/plotting
#      - post_processing=$PACKAGES_DIR/post_processing/ews_postprocessing
#      - source_gen=$PACKAGES_DIR/source_gen
#      - coordinator=$CI_PROJECT_DIR/coordinator
#      - coordinator_tests=$CI_PROJECT_DIR/tests
#      - export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests

      - echo $PYTHONPATH
#      - CI_PYTHON_PATH=$PYTHONPATH
#      - ls ../

#.full_fat_before_script:
#  before_script:
#      - source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/
#      - pip install coverage
#      - pwd
#      - CODE_DIR="/storage/app/EWS_prod/code/"
#      - flagdir=$CODE_DIR/flagdir
#      - epimodel=$CODE_DIR/epimodel
#      - advisory=$CODE_DIR/advisory_builder
#      - met_processing=$CODE_DIR/met_extractor_v2/met_data_extraction
#      - met_processor=$CODE_DIR/environmental_suitability/environmental_suitability
#      - plotting=$CODE_DIR/plotting/plotting
#      - post_processing=$CODE_DIR/post_processing/ews_postprocessing
#      - source_gen=$CODE_DIR/source_gen
#      - coordinator=$CODE_DIR/coordinator/coordinator
#      - coordinator_tests=$CODE_DIR/coordinator/tests
#      - export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests
#
#      - echo $PYTHONPATH
#      - CI_PYTHON_PATH=$PYTHONPATH

half_fat_tests:
  when: manual
  stage: half-fat
  extends: .half_fat_before_script
  image: lb584/ews_coordinator:latest

  script:
    - export PYTHONPATH="$CI_PYTHON_PATH"
    - echo $DEPLOY_ENVIRONMENT
#    - echo $PYTHONPATH
    - cd $CI_PROJECT_DIR/tests/unit/coordinator/
    - python3 -m coverage run -m unittest *
    - python -m coverage report
    - python -m coverage html -d $CI_PROJECT_DIR/coverage
    - cd $CI_PROJECT_DIR/tests/integration/partial/
    - python3 -m coverage run -m unittest test_advisory.py
    - python3 -m coverage run -m unittest test_deposition.py
    - python3 -m coverage run -m unittest test_env_suit.py
    - python3 -m coverage run -m unittest test_survey.py
    - python3 -m coverage report
    - python3 -m coverage html -d $CI_PROJECT_DIR/coverage
    - ls
  artifacts:
    paths:
      - $CI_PROJECT_DIR/coverage
    expire_in: 30 days

half_fat_epi_tests:
  when: manual
  stage: half-fat
  extends: .half_fat_before_script
  image: lb584/ews_coordinator:latest

  script:
    - cd $CI_PROJECT_DIR/tests/integration/partial/
    - python3 -m coverage run -m unittest test_epi.py
    - python3 -m coverage report
    - python3 -m coverage html -d $CI_PROJECT_DIR/coverage
    - ls
  artifacts:
    paths:
      - $CI_PROJECT_DIR/coverage
    expire_in: 30 days

full_fat_tests:
  stage: full-fat
  extends: .half_fat_before_script
  image: lb584/ews_coordinator:latest
  tags: ["ewas-production"]
  script:
#    - export PYTHONPATH="$CI_PYTHON_PATH"
    - echo $DEPLOY_ENVIRONMENT
#    - echo $PYTHONPATH
    - echo "this is a manual job"
    - cd $CI_PROJECT_DIR/tests/integration/full/
    - ls
#    - rm /storage/app/EWS_prod/regions/EastAfrica/workspace/DEPOSITION_20230126/STATUS_*
    - python3 full_test_deposition.py --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json --outdir /storage/app/EWS_prod/regions/EastAfrica/workspace/ --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json