ci: implement ci pipeline test jobs

This commit implements a matrix test job to run multiple
Terraform integration tests in parallel. It also implements
a cleanup script to ensure that failed test runs do not leave
infrastructure deployed which would cost money!

.gitlab-ci.yml

 include:
   - project: 'uis/devops/continuous-delivery/ci-templates'
     file: '/terraform-module.yml'
+    ref: "v3.6.1"
+  - project: 'uis/devops/continuous-delivery/ci-templates'
+    file: '/pre-commit.yml'
+    ref: "v3.6.1"
+
+variables:
+  LOGAN_IMAGE: registry.gitlab.developers.cam.ac.uk/uis/devops/infra/dockerimages/logan-terraform:1.6
+
+  # Disable the changelog check as it doesn't behave well with pre/beta releases. Also, the check is not required as
+  # we're using release-it for release automation which handles changelog auto-generation.
+  CHANGELOG_CHECK_DISABLED: 1
+
+# This is a workaround to allow the "tests" job matrix below to be manually triggered. Without this job, and the
+# associated "needs" dependency in the "tests" job, all tests would run on every push, which is undesirable given the
+# number of resources that they create. Instead, developers should manually trigger this job from the pipeline UI when
+# they require the test suite to run, for example as part of the MR review process.
+run_tests:
+  stage: test
+  when: manual
+  allow_failure: false
+  script: echo "Triggering test jobs..."
+
+.cleanup:
+  image: $LOGAN_IMAGE
+  script: ./tests/cleanup.sh
+  when: always
+  tags:
+    - $GKE_RUNNER_TAG
+
+pre-cleanup:
+  extends: .cleanup
+  stage: test
+  needs:
+    - run_tests
+
+tests:
+  stage: test
+  image: $LOGAN_IMAGE
+  variables:
+    GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: "terraform-deploy@infra-testing-int-e2395220.iam.gserviceaccount.com"
+  script: |
+    # This unsets the GOOGLE_APPLICATION_CREDENTIALS as it is not required but the logan-terraform images sets it.
+    unset GOOGLE_APPLICATION_CREDENTIALS
+
+    ./run_tests.sh -c -t "tests/$TEST_FILE"
+  needs:
+    - run_tests
+    - pre-cleanup
+  tags:
+    - $GKE_RUNNER_TAG
+  parallel:
+    # This matrix runs each of our test files in parallel targeting v4.x and v5.x of the Google Terraform provider
+    # separately as we support both. It also ensures that subnet CIDR ranges do not clash when testing the VPC
+    # Access/static egress IP configurations.
+    matrix:
+      - TEST_FILE:
+          - cloud_run_service.tftest.hcl
+          - cloudsql.tftest.hcl
+          - load_balancer.tftest.hcl
+          - pre_deploy_job.tftest.hcl
+        GOOGLE_PROVIDER_VERSION_CONSTRAINT:
+          - "> 4, < 5"
+          - "> 5, < 6"
+      - TEST_FILE:
+          - monitoring.tftest.hcl
+        GOOGLE_PROVIDER_VERSION_CONSTRAINT: "> 4, < 5"
+        TF_VAR_static_egress_ip_cidr_range: "10.0.0.0/28"
+        TF_VAR_test_ip_cidr_range: "10.0.0.16/28"
+      - TEST_FILE:
+          - monitoring.tftest.hcl
+        GOOGLE_PROVIDER_VERSION_CONSTRAINT: "> 5, < 6"
+        TF_VAR_static_egress_ip_cidr_range: "10.0.0.32/28"
+        TF_VAR_test_ip_cidr_range: "10.0.0.48/28"
+      - TEST_FILE:
+          - vpc_access.tftest.hcl
+        GOOGLE_PROVIDER_VERSION_CONSTRAINT: "> 4, < 5"
+        TF_VAR_static_egress_ip_cidr_range: "10.0.0.64/28"
+        TF_VAR_test_ip_cidr_range: "10.0.0.80/28"
+      - TEST_FILE:
+          - vpc_access.tftest.hcl
+        GOOGLE_PROVIDER_VERSION_CONSTRAINT: "> 5, < 6"
+        TF_VAR_static_egress_ip_cidr_range: "10.0.0.96/28"
+        TF_VAR_test_ip_cidr_range: "10.0.0.112/28"
+
+post-cleanup:
+  extends: .cleanup
+  stage: review

tests/cleanup.sh

+#! /usr/bin/env bash
+
+set -e
+
+current_verbosity=$(gcloud config get core/verbosity)
+gcloud config set core/verbosity error
+
+cleanup() {
+    gcloud config unset auth/impersonate_service_account
+    gcloud config set core/verbosity "$current_verbosity"
+}
+
+trap 'cleanup' EXIT INT TERM
+
+TEST_PREFIX="test-rapp"
+GCP_PROJECT="infra-testing-int-e2395220"
+GCP_PROJECT_META="infra-testing-meta-21f09a44"
+GCP_REGION="europe-west2"
+GCP_SERVICE_ACCOUNT="terraform-deploy@infra-testing-int-e2395220.iam.gserviceaccount.com"
+
+gcloud config set auth/impersonate_service_account $GCP_SERVICE_ACCOUNT
+
+echo "Cleaning up Cloud Run services..."
+mapfile -t services < <(
+    gcloud --project="$GCP_PROJECT" run services --region="$GCP_REGION" list \
+        --filter="metadata.name ~ ${TEST_PREFIX}.*" --format="value(metadata.name)"
+)
+
+for service in "${services[@]}"; do
+    echo "Removing Cloud Run service '${service}'"
+    gcloud --project="$GCP_PROJECT" run services --region="$GCP_REGION" delete "$service" --quiet
+done
+
+echo "Cleaning up IAM service accounts..."
+mapfile -t service_accounts < <(
+    gcloud --project="$GCP_PROJECT" iam service-accounts list \
+        --filter="email ~ ${TEST_PREFIX}[0-9a-fA-F]+?-run|${TEST_PREFIX}[0-9a-fA-F]+?-uptime" \
+        --format="value(email)"
+)
+
+for account in "${service_accounts[@]}"; do
+    gcloud --project="$GCP_PROJECT" iam service-accounts delete "$account" --quiet
+done
+
+echo "Cleaning up Cloud Functions..."
+mapfile -t functions < <(
+    gcloud --project="$GCP_PROJECT" functions list \
+        --filter="name ~ .*${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for function in "${functions[@]}"; do
+    gcloud --project="$GCP_PROJECT" functions delete --region="$GCP_REGION" "$function" --quiet
+done
+
+echo "Cleaning up Cloud Storage buckets..."
+mapfile -t buckets < <(
+    gcloud --project="$GCP_PROJECT" storage buckets list \
+        --filter="name ~ ${TEST_PREFIX::8}-uptime" --format="value(storage_url)"
+)
+
+for bucket in "${buckets[@]}"; do
+    gcloud --project="$GCP_PROJECT" storage rm -r "$bucket" --quiet
+done
+
+echo "Cleaning up Cloud Monitoring resources..."
+mapfile -t alert_policies < <(
+    gcloud alpha --project="$GCP_PROJECT" monitoring policies list \
+        --filter="displayName ~ Uptime\scheck\sfor\s${TEST_PREFIX}[0-9a-fA-F]+?-.*|SSL\sexpiry\scheck\sfor\s${TEST_PREFIX}[0-9a-fA-F]+?-.*" \
+        --format="value(name)"
+)
+
+for policy in "${alert_policies[@]}"; do
+    gcloud alpha monitoring policies delete "$policy" --quiet
+done
+
+mapfile -t alert_policies_meta < <(
+    gcloud alpha --project="$GCP_PROJECT_META" monitoring policies list \
+        --filter="displayName ~ Uptime\scheck\sfor\s${TEST_PREFIX}[0-9a-fA-F]+?-.*|SSL\sexpiry\scheck\sfor\s${TEST_PREFIX}[0-9a-fA-F]+?-.*" \
+        --format="value(name)"
+)
+
+for policy_meta in "${alert_policies_meta[@]}"; do
+    gcloud alpha monitoring policies delete "$policy_meta" --quiet
+done
+
+mapfile -t uptime_checks < <(
+    gcloud --project="$GCP_PROJECT" monitoring uptime list-configs \
+        --filter="displayName ~ ${TEST_PREFIX}[0-9a-fA-F]+?-.*" --format="value(name)"
+)
+
+for check in "${uptime_checks[@]}"; do
+    gcloud monitoring uptime delete "$check" --quiet
+done
+
+echo "Cleaning up Cloud Run jobs..."
+mapfile -t jobs < <(
+    gcloud --project="$GCP_PROJECT" run jobs list \
+        --filter="metadata.name ~ ${TEST_PREFIX}.*" --format="value(metadata.name)"
+)
+
+for job in "${jobs[@]}"; do
+    gcloud --project="$GCP_PROJECT" run jobs --region="$GCP_REGION" delete "$job" --quiet
+done
+
+echo "Cleaning up load balancer resources..."
+mapfile -t http_proxies < <(
+    gcloud --project="$GCP_PROJECT" compute target-http-proxies list \
+        --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for http_proxy in "${http_proxies[@]}"; do
+    gcloud --project="$GCP_PROJECT" compute target-http-proxies delete "$http_proxy" --global --quiet
+done
+
+mapfile -t https_proxies < <(
+    gcloud --project="$GCP_PROJECT" compute target-https-proxies list \
+        --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for https_proxy in "${https_proxies[@]}"; do
+    gcloud --project="$GCP_PROJECT" compute target-https-proxies delete "$https_proxy" --global --quiet
+done
+
+mapfile -t ssl_certs < <(
+    gcloud --project="$GCP_PROJECT" compute ssl-certificates list \
+        --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for cert in "${ssl_certs[@]}"; do
+    gcloud --project="$GCP_PROJECT" compute ssl-certificates delete "$cert" --global --quiet
+done
+
+mapfile -t url_maps < <(
+    gcloud --project="$GCP_PROJECT" compute url-maps list \
+        --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for url_map in "${url_maps[@]}"; do
+    gcloud --project="$GCP_PROJECT" compute url-maps delete "$url_map" --global --quiet
+done
+
+mapfile -t backend_services < <(
+    gcloud --project="$GCP_PROJECT" compute backend-services list \
+        --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for service in "${backend_services[@]}"; do
+    gcloud --project="$GCP_PROJECT" compute backend-services delete "$service" --global --quiet
+done
+
+mapfile -t serverless_negs < <(
+    gcloud --project="$GCP_PROJECT" compute network-endpoint-groups list \
+        --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for neg in "${serverless_negs[@]}"; do
+    gcloud --project="$GCP_PROJECT" compute network-endpoint-groups delete "$neg" --region="$GCP_REGION" --quiet
+done
+
+mapfile -t ssl_policies < <(
+    gcloud --project="$GCP_PROJECT" compute ssl-policies list \
+        --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for ssl_policy in "${ssl_policies[@]}"; do
+    gcloud --project="$GCP_PROJECT" compute ssl-policies delete "$ssl_policy" --global --quiet
+done
+
+echo "Cleaning up network resources..."
+mapfile -t connectors < <(
+    gcloud --project="$GCP_PROJECT" compute networks vpc-access connectors list --region="$GCP_REGION" \
+        --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for conn in "${connectors[@]}"; do
+    gcloud --project="$GCP_PROJECT" compute networks vpc-access connectors delete "$conn" \
+        --region="$GCP_REGION" --quiet
+done
+
+mapfile -t routers < <(
+    gcloud --project="$GCP_PROJECT" compute routers list --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for router in "${routers[@]}"; do
+    gcloud --project="$GCP_PROJECT" compute routers delete "$router" --region="$GCP_REGION" --quiet
+done
+
+mapfile -t addresses < <(
+    gcloud --project="$GCP_PROJECT" compute addresses list --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)" \
+        --global
+)
+
+for address in "${addresses[@]}"; do
+    gcloud --project="$GCP_PROJECT" compute addresses delete "$address" --global --quiet
+done
+
+mapfile -t subnets < <(
+    gcloud --project="$GCP_PROJECT" compute networks subnets list \
+        --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for subnet in "${subnets[@]}"; do
+    gcloud --project="$GCP_PROJECT" compute networks subnets delete "$subnet" --region="$GCP_REGION" --quiet
+done
+
+echo "Cleaning up test setup resources..."
+mapfile -t instances < <(
+    gcloud --project="$GCP_PROJECT" sql instances list --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for instance in "${instances[@]}"; do
+    gcloud --project="$GCP_PROJECT" sql instances delete "$instance" --quiet
+done
+
+mapfile -t secrets < <(
+    gcloud --project="$GCP_PROJECT" secrets list --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for secret in "${secrets[@]}"; do
+    gcloud --project="$GCP_PROJECT" secrets delete "$secret" --quiet
+done