From a840ce9449b25e5f206ebd40fc5d979d3ea3c57a Mon Sep 17 00:00:00 2001
From: Ryan Kowalewski <rk725@cam.ac.uk>
Date: Thu, 14 Dec 2023 15:29:56 +0000
Subject: [PATCH] ci: implement ci pipeline test jobs
This commit implements a matrix test job to run multiple
Terraform integration tests in parallel. It also implements
a cleanup script to ensure that failed test runs do not leave
infrastructure deployed which would cost money!
---
.gitlab-ci.yml | 87 +++++++++++++++++++
tests/cleanup.sh | 221 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 308 insertions(+)
create mode 100755 tests/cleanup.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6416c68..27bdead 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,3 +1,90 @@
include:
- project: 'uis/devops/continuous-delivery/ci-templates'
file: '/terraform-module.yml'
+ ref: "v3.6.1"
+ - project: 'uis/devops/continuous-delivery/ci-templates'
+ file: '/pre-commit.yml'
+ ref: "v3.6.1"
+
+variables:
+ LOGAN_IMAGE: registry.gitlab.developers.cam.ac.uk/uis/devops/infra/dockerimages/logan-terraform:1.6
+
+ # Disable the changelog check as it doesn't behave well with pre/beta releases. Also, the check is not required as
+ # we're using release-it for release automation which handles changelog auto-generation.
+ CHANGELOG_CHECK_DISABLED: 1
+
+# This is a workaround to allow the "tests" job matrix below to be manually triggered. Without this job, and the
+# associated "needs" dependency in the "tests" job, all tests would run on every push, which is undesirable given the
+# number of resources that they create. Instead, developers should manually trigger this job from the pipeline UI when
+# they require the test suite to run, for example as part of the MR review process.
+run_tests:
+ stage: test
+ when: manual
+ allow_failure: false
+ script: echo "Triggering test jobs..."
+
+.cleanup:
+ image: $LOGAN_IMAGE
+ script: ./tests/cleanup.sh
+ when: always
+ tags:
+ - $GKE_RUNNER_TAG
+
+pre-cleanup:
+ extends: .cleanup
+ stage: test
+ needs:
+ - run_tests
+
+tests:
+ stage: test
+ image: $LOGAN_IMAGE
+ variables:
+ GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: "terraform-deploy@infra-testing-int-e2395220.iam.gserviceaccount.com"
+ script: |
+ # This unsets the GOOGLE_APPLICATION_CREDENTIALS as it is not required but the logan-terraform images sets it.
+ unset GOOGLE_APPLICATION_CREDENTIALS
+
+ ./run_tests.sh -c -t "tests/$TEST_FILE"
+ needs:
+ - run_tests
+ - pre-cleanup
+ tags:
+ - $GKE_RUNNER_TAG
+ parallel:
+ # This matrix runs each of our test files in parallel targeting v4.x and v5.x of the Google Terraform provider
+ # separately as we support both. It also ensures that subnet CIDR ranges do not clash when testing the VPC
+ # Access/static egress IP configurations.
+ matrix:
+ - TEST_FILE:
+ - cloud_run_service.tftest.hcl
+ - cloudsql.tftest.hcl
+ - load_balancer.tftest.hcl
+ - pre_deploy_job.tftest.hcl
+ GOOGLE_PROVIDER_VERSION_CONSTRAINT:
+ - "> 4, < 5"
+ - "> 5, < 6"
+ - TEST_FILE:
+ - monitoring.tftest.hcl
+ GOOGLE_PROVIDER_VERSION_CONSTRAINT: "> 4, < 5"
+ TF_VAR_static_egress_ip_cidr_range: "10.0.0.0/28"
+ TF_VAR_test_ip_cidr_range: "10.0.0.16/28"
+ - TEST_FILE:
+ - monitoring.tftest.hcl
+ GOOGLE_PROVIDER_VERSION_CONSTRAINT: "> 5, < 6"
+ TF_VAR_static_egress_ip_cidr_range: "10.0.0.32/28"
+ TF_VAR_test_ip_cidr_range: "10.0.0.48/28"
+ - TEST_FILE:
+ - vpc_access.tftest.hcl
+ GOOGLE_PROVIDER_VERSION_CONSTRAINT: "> 4, < 5"
+ TF_VAR_static_egress_ip_cidr_range: "10.0.0.64/28"
+ TF_VAR_test_ip_cidr_range: "10.0.0.80/28"
+ - TEST_FILE:
+ - vpc_access.tftest.hcl
+ GOOGLE_PROVIDER_VERSION_CONSTRAINT: "> 5, < 6"
+ TF_VAR_static_egress_ip_cidr_range: "10.0.0.96/28"
+ TF_VAR_test_ip_cidr_range: "10.0.0.112/28"
+
+post-cleanup:
+ extends: .cleanup
+ stage: review
diff --git a/tests/cleanup.sh b/tests/cleanup.sh
new file mode 100755
index 0000000..a7ba708
--- /dev/null
+++ b/tests/cleanup.sh
@@ -0,0 +1,221 @@
+#! /usr/bin/env bash
+
+set -e
+
+current_verbosity=$(gcloud config get core/verbosity)
+gcloud config set core/verbosity error
+
+cleanup() {
+ gcloud config unset auth/impersonate_service_account
+ gcloud config set core/verbosity "$current_verbosity"
+}
+
+trap 'cleanup' EXIT INT TERM
+
+TEST_PREFIX="test-rapp"
+GCP_PROJECT="infra-testing-int-e2395220"
+GCP_PROJECT_META="infra-testing-meta-21f09a44"
+GCP_REGION="europe-west2"
+GCP_SERVICE_ACCOUNT="terraform-deploy@infra-testing-int-e2395220.iam.gserviceaccount.com"
+
+gcloud config set auth/impersonate_service_account $GCP_SERVICE_ACCOUNT
+
+echo "Cleaning up Cloud Run services..."
+mapfile -t services < <(
+ gcloud --project="$GCP_PROJECT" run services --region="$GCP_REGION" list \
+ --filter="metadata.name ~ ${TEST_PREFIX}.*" --format="value(metadata.name)"
+)
+
+for service in "${services[@]}"; do
+ echo "Removing Cloud Run service '${service}'"
+ gcloud --project="$GCP_PROJECT" run services --region="$GCP_REGION" delete "$service" --quiet
+done
+
+echo "Cleaning up IAM service accounts..."
+mapfile -t service_accounts < <(
+ gcloud --project="$GCP_PROJECT" iam service-accounts list \
+ --filter="email ~ ${TEST_PREFIX}[0-9a-fA-F]+?-run|${TEST_PREFIX}[0-9a-fA-F]+?-uptime" \
+ --format="value(email)"
+)
+
+for account in "${service_accounts[@]}"; do
+ gcloud --project="$GCP_PROJECT" iam service-accounts delete "$account" --quiet
+done
+
+echo "Cleaning up Cloud Functions..."
+mapfile -t functions < <(
+ gcloud --project="$GCP_PROJECT" functions list \
+ --filter="name ~ .*${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for function in "${functions[@]}"; do
+ gcloud --project="$GCP_PROJECT" functions delete --region="$GCP_REGION" "$function" --quiet
+done
+
+echo "Cleaning up Cloud Storage buckets..."
+mapfile -t buckets < <(
+ gcloud --project="$GCP_PROJECT" storage buckets list \
+ --filter="name ~ ${TEST_PREFIX::8}-uptime" --format="value(storage_url)"
+)
+
+for bucket in "${buckets[@]}"; do
+ gcloud --project="$GCP_PROJECT" storage rm -r "$bucket" --quiet
+done
+
+echo "Cleaning up Cloud Monitoring resources..."
+mapfile -t alert_policies < <(
+ gcloud alpha --project="$GCP_PROJECT" monitoring policies list \
+ --filter="displayName ~ Uptime\scheck\sfor\s${TEST_PREFIX}[0-9a-fA-F]+?-.*|SSL\sexpiry\scheck\sfor\s${TEST_PREFIX}[0-9a-fA-F]+?-.*" \
+ --format="value(name)"
+)
+
+for policy in "${alert_policies[@]}"; do
+ gcloud alpha monitoring policies delete "$policy" --quiet
+done
+
+mapfile -t alert_policies_meta < <(
+ gcloud alpha --project="$GCP_PROJECT_META" monitoring policies list \
+ --filter="displayName ~ Uptime\scheck\sfor\s${TEST_PREFIX}[0-9a-fA-F]+?-.*|SSL\sexpiry\scheck\sfor\s${TEST_PREFIX}[0-9a-fA-F]+?-.*" \
+ --format="value(name)"
+)
+
+for policy_meta in "${alert_policies_meta[@]}"; do
+ gcloud alpha monitoring policies delete "$policy_meta" --quiet
+done
+
+mapfile -t uptime_checks < <(
+ gcloud --project="$GCP_PROJECT" monitoring uptime list-configs \
+ --filter="displayName ~ ${TEST_PREFIX}[0-9a-fA-F]+?-.*" --format="value(name)"
+)
+
+for check in "${uptime_checks[@]}"; do
+ gcloud monitoring uptime delete "$check" --quiet
+done
+
+echo "Cleaning up Cloud Run jobs..."
+mapfile -t jobs < <(
+ gcloud --project="$GCP_PROJECT" run jobs list \
+ --filter="metadata.name ~ ${TEST_PREFIX}.*" --format="value(metadata.name)"
+)
+
+for job in "${jobs[@]}"; do
+ gcloud --project="$GCP_PROJECT" run jobs --region="$GCP_REGION" delete "$job" --quiet
+done
+
+echo "Cleaning up load balancer resources..."
+mapfile -t http_proxies < <(
+ gcloud --project="$GCP_PROJECT" compute target-http-proxies list \
+ --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for http_proxy in "${http_proxies[@]}"; do
+ gcloud --project="$GCP_PROJECT" compute target-http-proxies delete "$http_proxy" --global --quiet
+done
+
+mapfile -t https_proxies < <(
+ gcloud --project="$GCP_PROJECT" compute target-https-proxies list \
+ --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for https_proxy in "${https_proxies[@]}"; do
+ gcloud --project="$GCP_PROJECT" compute target-https-proxies delete "$https_proxy" --global --quiet
+done
+
+mapfile -t ssl_certs < <(
+ gcloud --project="$GCP_PROJECT" compute ssl-certificates list \
+ --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for cert in "${ssl_certs[@]}"; do
+ gcloud --project="$GCP_PROJECT" compute ssl-certificates delete "$cert" --global --quiet
+done
+
+mapfile -t url_maps < <(
+ gcloud --project="$GCP_PROJECT" compute url-maps list \
+ --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for url_map in "${url_maps[@]}"; do
+ gcloud --project="$GCP_PROJECT" compute url-maps delete "$url_map" --global --quiet
+done
+
+mapfile -t backend_services < <(
+ gcloud --project="$GCP_PROJECT" compute backend-services list \
+ --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for service in "${backend_services[@]}"; do
+ gcloud --project="$GCP_PROJECT" compute backend-services delete "$service" --global --quiet
+done
+
+mapfile -t serverless_negs < <(
+ gcloud --project="$GCP_PROJECT" compute network-endpoint-groups list \
+ --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for neg in "${serverless_negs[@]}"; do
+ gcloud --project="$GCP_PROJECT" compute network-endpoint-groups delete "$neg" --region="$GCP_REGION" --quiet
+done
+
+mapfile -t ssl_policies < <(
+ gcloud --project="$GCP_PROJECT" compute ssl-policies list \
+ --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for ssl_policy in "${ssl_policies[@]}"; do
+ gcloud --project="$GCP_PROJECT" compute ssl-policies delete "$ssl_policy" --global --quiet
+done
+
+echo "Cleaning up network resources..."
+mapfile -t connectors < <(
+ gcloud --project="$GCP_PROJECT" compute networks vpc-access connectors list --region="$GCP_REGION" \
+ --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for conn in "${connectors[@]}"; do
+ gcloud --project="$GCP_PROJECT" compute networks vpc-access connectors delete "$conn" \
+ --region="$GCP_REGION" --quiet
+done
+
+mapfile -t routers < <(
+ gcloud --project="$GCP_PROJECT" compute routers list --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for router in "${routers[@]}"; do
+ gcloud --project="$GCP_PROJECT" compute routers delete "$router" --region="$GCP_REGION" --quiet
+done
+
+mapfile -t addresses < <(
+ gcloud --project="$GCP_PROJECT" compute addresses list --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)" \
+ --global
+)
+
+for address in "${addresses[@]}"; do
+ gcloud --project="$GCP_PROJECT" compute addresses delete "$address" --global --quiet
+done
+
+mapfile -t subnets < <(
+ gcloud --project="$GCP_PROJECT" compute networks subnets list \
+ --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for subnet in "${subnets[@]}"; do
+ gcloud --project="$GCP_PROJECT" compute networks subnets delete "$subnet" --region="$GCP_REGION" --quiet
+done
+
+echo "Cleaning up test setup resources..."
+mapfile -t instances < <(
+ gcloud --project="$GCP_PROJECT" sql instances list --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for instance in "${instances[@]}"; do
+ gcloud --project="$GCP_PROJECT" sql instances delete "$instance" --quiet
+done
+
+mapfile -t secrets < <(
+ gcloud --project="$GCP_PROJECT" secrets list --filter="name ~ ${TEST_PREFIX}.*" --format="value(name)"
+)
+
+for secret in "${secrets[@]}"; do
+ gcloud --project="$GCP_PROJECT" secrets delete "$secret" --quiet
+done
--
GitLab