variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" PACKAGES_DIR: "$CI_PROJECT_DIR/packages/" OUTPUT_DIR: "$CI_PROJECT_DIR/outputs/" TEST_WORKSPACE_DIR: "/storage/app/EWS_prod/regions/EastAfrica/test_workspace/" CI_PYTHON_PATH: "NOT_SET" FULL_FAT_RUN_DATE: value: "default" options: - "default" - "today" - "yesterday" - "custom" description: "Full-fat tests will pull data from the deployment environment. For robustness, the default option is a pre-selected date we know works. Select \"today\" or \"yesterday\" to test with recent live data. If \"custom\" is selected, manually enter the key value pair using the format in this example: CUSTOM_RUN_DATE 20230115" CUSTOM_RUN_DATE: "NOT_SET" FULL_FAT_COPY_OUTPUT: value: "false" options: - "true" - "false" description: "Select \"true\" to make a copy of the full-fat integration test output. Outputs will be copied to the production machine at /storage/app/EWS_prod/regions/EastAfrica/test_workspace" cache: paths: - .cache/pip - venv/ stages: - half-fat - full-fat .half_fat_before_script: before_script: - python3 -V # Print out python version for debugging - pip install coverage - echo $PACKAGES_DIR - mkdir $PACKAGES_DIR - mkdir $OUTPUT_DIR - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-source-generation.git" $PACKAGES_DIR/source_gen - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_plotting.git" $PACKAGES_DIR/plotting - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/flagdir.git" $PACKAGES_DIR/flagdir - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-epimodel.git" $PACKAGES_DIR/epimodel - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-advisory-builder.git" $PACKAGES_DIR/advisory_builder - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_environmental_suitability_v2.git" $PACKAGES_DIR/environmental_suitability - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing - git clone "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_met_data_extraction.git" $PACKAGES_DIR/met_extractor_v2 - flagdir=$PACKAGES_DIR/flagdir - epimodel=$PACKAGES_DIR/epimodel - advisory=$PACKAGES_DIR/advisory_builder - met_processing=$PACKAGES_DIR/met_extractor_v2/met_data_extraction - met_processor=$PACKAGES_DIR/environmental_suitability/environmental_suitability - plotting=$PACKAGES_DIR/plotting/plotting - post_processing=$PACKAGES_DIR/post_processing/ews_postprocessing - source_gen=$PACKAGES_DIR/source_gen - coordinator=$CI_PROJECT_DIR/coordinator - coordinator_tests=$CI_PROJECT_DIR/tests - export PYTHONPATH=$PYTHONPATH:$coordinator:$coordinator_tests - echo $PYTHONPATH - ls $PACKAGES_DIR - source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/ #.full_fat_before_script: # before_script: # - source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/ # - pip install coverage # - pwd # - CODE_DIR="/storage/app/EWS_prod/code/" # - flagdir=$CODE_DIR/flagdir # - epimodel=$CODE_DIR/epimodel # - advisory=$CODE_DIR/advisory_builder # - met_processing=$CODE_DIR/met_extractor_v2/met_data_extraction # - met_processor=$CODE_DIR/environmental_suitability/environmental_suitability # - plotting=$CODE_DIR/plotting/plotting # - post_processing=$CODE_DIR/post_processing/ews_postprocessing # - source_gen=$CODE_DIR/source_gen # - coordinator=$CODE_DIR/coordinator/coordinator # - coordinator_tests=$CODE_DIR/coordinator/tests # - export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests # # - echo $PYTHONPATH # - CI_PYTHON_PATH=$PYTHONPATH half_fat_tests: when: manual stage: half-fat extends: .half_fat_before_script image: lb584/ews_coordinator:latest script: - export PYTHONPATH="$CI_PYTHON_PATH" - echo $FULL_FAT_RUN_DATE # - echo $PYTHONPATH - cd $CI_PROJECT_DIR/tests/unit/coordinator/ - python3 -m coverage run -m unittest * - python -m coverage report - python -m coverage html -d $CI_PROJECT_DIR/coverage - cd $CI_PROJECT_DIR/tests/integration/partial/ - python3 -m coverage run -m unittest test_advisory.py - python3 -m coverage run -m unittest test_deposition.py - python3 -m coverage run -m unittest test_env_suit.py - python3 -m coverage run -m unittest test_survey.py - python3 -m coverage report - python3 -m coverage html -d $CI_PROJECT_DIR/coverage - ls artifacts: paths: - $CI_PROJECT_DIR/coverage expire_in: 10 days half_fat_epi_tests: when: manual stage: half-fat extends: .half_fat_before_script image: lb584/ews_coordinator:latest script: - cd $CI_PROJECT_DIR/tests/integration/partial/ - python3 -m coverage run -m unittest test_epi.py - python3 -m coverage report - python3 -m coverage html -d $CI_PROJECT_DIR/coverage - ls artifacts: paths: - $CI_PROJECT_DIR/coverage expire_in: 10 days full_fat_tests: when: manual stage: full-fat extends: .half_fat_before_script image: lb584/ews_coordinator:latest tags: ["ewas-production"] script: - > echo $FULL_FAT_RUN_DATE echo $CUSTOM_RUN_DATE #echo $PYTHONPATH #echo "this is a manual job" cd $CI_PROJECT_DIR/tests/integration/full/ #ls #rm /storage/app/EWS_prod/regions/EastAfrica/workspace/DEPOSITION_20230126/STATUS_* #mkdir -p $OUTPUT_DIR/deposition_123445 #touch $OUTPUT_DIR/deposition_123445/log.txt #echo "stuff" > $OUTPUT_DIR/deposition_123445/log.txt #mkdir $OUTPUT_DIR/survey_123445 #touch $OUTPUT_DIR/survey_123445/log.txt #echo "stuff2" > $OUTPUT_DIR/survey_123445/log.txt #echo $TEST_WORKSPACE_DIR python3 full_test_deposition.py --config /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json --outdir $OUTPUT_DIR --email_cred /storage/app/EWS_prod/envs/credentials/Cred_gmail.json --run_date_type $FULL_FAT_RUN_DATE --custom_run_date $CUSTOM_RUN_DATE if [[ "$FULL_FAT_COPY_OUTPUT" == "true" ]]; then date=$(date '+%Y-%m-%d-%H%M'); mkdir ${TEST_WORKSPACE_DIR}/integration/full/${date}; cp -rv $OUTPUT_DIR/* ${TEST_WORKSPACE_DIR}/integration/full/${date}; fi artifacts: paths: - $OUTPUT_DIR/*/log.txt expire_in: 10 days