experimenting with pipeline vartiables

.gitlab-ci.yml

@@ -4,13 +4,17 @@ variables:
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   PACKAGES_DIR: "$CI_PROJECT_DIR/packages/"
   CI_PYTHON_PATH: "NOT_SET"
-  DEPLOY_ENVIRONMENT:
-    value: "staging"
+  FULL_FAT_RUN_DATE:
+    value: "today"
     options:
-      - "production"
-      - "staging"
-      - "canary"
-    description: "The deployment target. Set to 'staging' by default."
+      - "today"
+      - "yesterday"
+      - "custom"
+    description: "Full-fat tests will pull data from the deployment environment. As with the production pipeline, not 
+    specifying a date will result in todays date being used. If \"custom\" is selected, manually enter the key value 
+    pair using the format in this example: CUSTOM_RUN_DATE 20230115"
+
+  CUSTOM_RUN_DATE: "NOT_SET"
 
 # Pip's cache doesn't store the python packages
 # https://pip.pypa.io/en/stable/reference/pip_install/#caching
@@ -42,22 +46,6 @@ stages:
       - ls $PACKAGES_DIR
       - source /storage/app/miniconda3/bin/activate /storage/app/EWS_prod/envs/conda/py3EWS/
       - pip install coverage
-      - pwd
-#      - flagdir=$PACKAGES_DIR/flagdir
-#      - epimodel=$PACKAGES_DIR/epimodel
-#      - advisory=$PACKAGES_DIR/advisory_builder
-#      - met_processing=$PACKAGES_DIR/met_extractor_v2/met_data_extraction
-#      - met_processor=$PACKAGES_DIR/environmental_suitability/environmental_suitability
-#      - plotting=$PACKAGES_DIR/plotting/plotting
-#      - post_processing=$PACKAGES_DIR/post_processing/ews_postprocessing
-#      - source_gen=$PACKAGES_DIR/source_gen
-#      - coordinator=$CI_PROJECT_DIR/coordinator
-#      - coordinator_tests=$CI_PROJECT_DIR/tests
-#      - export PYTHONPATH=$PYTHONPATH:$flagdir:$epimodel:$advisory:$met_processing:$met_processor:$plotting:$source_gen:$post_processing:$coordinator:$coordinator_tests
-
-      - echo $PYTHONPATH
-#      - CI_PYTHON_PATH=$PYTHONPATH
-#      - ls ../
 
 #.full_fat_before_script:
 #  before_script:
@@ -88,7 +76,7 @@ half_fat_tests:
 
   script:
     - export PYTHONPATH="$CI_PYTHON_PATH"
-    - echo $DEPLOY_ENVIRONMENT
+    - echo $FULL_FAT_RUN_DATE
 #    - echo $PYTHONPATH
     - cd $CI_PROJECT_DIR/tests/unit/coordinator/
     - python3 -m coverage run -m unittest *
@@ -125,13 +113,15 @@ half_fat_epi_tests:
     expire_in: 30 days
 
 full_fat_tests:
+  when: manual
   stage: full-fat
   extends: .half_fat_before_script
   image: lb584/ews_coordinator:latest
   tags: ["ewas-production"]
   script:
 #    - export PYTHONPATH="$CI_PYTHON_PATH"
-    - echo $DEPLOY_ENVIRONMENT
+    - echo $FULL_FAT_RUN_DATE
+    - echo $CUSTOM_RUN_DATE
 #    - echo $PYTHONPATH
     - echo "this is a manual job"
     - cd $CI_PROJECT_DIR/tests/integration/full/