Merge branch 'terraform-improvements-dev-deploy' into 'master'

Allow deployment of feature branch images via the terraform pipeline

See merge request !56

CHANGELOG.md

@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.6.0] - 2023-07-24
+
+### Added
+
+- terraform-pipeline: Allow the tfsec job to exclude specific checks. This is useful to exclude a
+  default list of checks that we don't follow in the boilerplate.
+
+### Fixed
+
+- artifact-registry: Allow manual push to registry for non-default branches. This is useful for
+  testing container images from feature branches. However, to avoid littering the artifact registry
+  with endless feature branch containers, it is configured as a manual job.
+- terraform-pipeline: Allow the development apply job to fail. This is currently the only way to
+  allow the whole pipeline to show as succeeded if the (optional) development apply job has not been
+  triggered. Otherwise, the pipeline shows as blocked, which is confusing. See the following issue
+  for context
+  [https://gitlab.com/gitlab-org/gitlab/-/issues/249524](https://gitlab.com/gitlab-org/gitlab/-/issues/249524).
+
 ## [2.5.0] - 2023-07-17
 
 ### Added

auto-devops/artifact-registry.yml

@@ -24,8 +24,7 @@
 #   $ARTIFACT_REGISTRY_DOCKER_REPOSITORY. The gke_ci_run service account configured by the gitlab-runner-infrastructure
 #   code must have permission to impersonate this service account.
 
-artifact-registry-push:
-  stage: production
+.artifact-registry-base:
   image: registry.gitlab.developers.cam.ac.uk/uis/devops/infra/dockerimages/gcloud-docker:latest
   services:
     - docker:24-dind
@@ -48,7 +47,7 @@ artifact-registry-push:
     fi
 
     gitlab_image="$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
-    
+
     # This uses "${string%substring}" to remove everything after the "@" (and the "@" itself) from the image digest
     # leaving a the hash prefix (e.g. "898d68000bd66376f44f0c1bb1bd73e68f2f0daa").
     artifact_registry_image="$CI_ARTIFACT_REGISTRY_REPOSITORY:${CI_APPLICATION_TAG%@*}"
@@ -65,10 +64,43 @@ artifact-registry-push:
     docker tag $gitlab_image $artifact_registry_image_latest
     docker push $artifact_registry_image
     docker push $artifact_registry_image_latest
+
+    # ANSI colour escape code, just for fun!
+    purple='\033[1;35m'
+    no_colour='\033[0m'
+    echo -e "$purple\nARTIFACT REGISTRY IMAGE TAG:\n\n$artifact_registry_image$no_colour"
   tags:
     - $GKE_RUNNER_TAG
+
+.only-when-artifact-registry-enabled:
+  - if: $GKE_RUNNER_TAG == null || $ARTIFACT_REGISTRY_SERVICE_ACCOUNT == null || $ARTIFACT_REGISTRY_DOCKER_REPOSITORY == null
+    when: never
+
+artifact-registry-push:
+  extends: .artifact-registry-base
+  stage: production
   rules:
-    - if: $GKE_RUNNER_TAG == null || $ARTIFACT_REGISTRY_SERVICE_ACCOUNT == null || $ARTIFACT_REGISTRY_DOCKER_REPOSITORY == null
+    - if: $DISABLE_ARTIFACT_REGISTRY_PUSH
       when: never
+    - !reference [.only-when-artifact-registry-enabled]
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_COMMIT_TAG
+
+artifact-registry-push-dev:
+  extends: .artifact-registry-base
+  stage: review
+  rules:
+    - if: $DISABLE_ARTIFACT_REGISTRY_PUSH
+      when: never
+    - !reference [.only-when-artifact-registry-enabled]
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: never
+    - if: $CI_COMMIT_TAG
+      when: never
+    - if: $CI_COMMIT_BRANCH
+      when: manual
+      # This is currently the only way to allow the pipeline to succeed if the optional manual push job was
+      # not triggered. See the following issue for context - https://gitlab.com/gitlab-org/gitlab/-/issues/249524.
+      allow_failure: true
+  needs:
+    - build

auto-devops/terraform-pipeline.yml

@@ -222,17 +222,29 @@ tflint:
     - $GKE_RUNNER_TAG
   needs: []
 
-# tfsec (https://aquasecurity.github.io/tfsec) is a comprehensive static analysis tool with many security
-# related checks for multiple cloud providers. This job uploads a junit report of the test results which is viewable
-# in the merge request UI. To ignore specific checks see - https://aquasecurity.github.io/tfsec/v1.28.1/guides/configuration/ignores/
+# tfsec (https://aquasecurity.github.io/tfsec) is a comprehensive static analysis tool with many security related checks
+# for multiple cloud providers. This job uploads a junit report of the test results which is viewable in the merge
+# request UI. To ignore specific checks see -
+# https://aquasecurity.github.io/tfsec/v1.28.1/guides/configuration/ignores/. By default, we're ignoring certain checks
+# which we have decided to allow due to our boilerplate template design.
 tfsec:
   stage: test
   image:
     name: aquasec/tfsec:latest
     entrypoint: [""]
+  variables:
+    TFSEC_EXCLUDE: "google-storage-bucket-encryption-customer-key,\
+      google-compute-enable-vpc-flow-logs,\
+      google-sql-enable-pg-temp-file-logging,\
+      google-sql-no-public-access,\
+      google-sql-pg-log-checkpoints,\
+      google-sql-pg-log-connections,\
+      google-sql-pg-log-disconnections,\
+      google-sql-pg-log-lock-waits"
   script: |
     mkdir ${TF_DATA_DIR}
-    tfsec --force-all-dirs --include-passed --format lovely,junit --out ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-tfsec
+    tfsec --force-all-dirs --include-passed --format lovely,junit \
+      --exclude "$TFSEC_EXCLUDE" --out ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-tfsec
   rules:
     - if: $TFSEC_DISABLED
       when: never
@@ -266,7 +278,12 @@ terraform-apply-development:
   rules:
     - if: $TERRAFORM_APPLY_DEVELOPMENT_DISABLED
       when: never
-    - !reference [.terraform-apply, rules]
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: manual
+      # This is currently the only way to allow the pipeline to succeed if the optional manual development apply job was
+      # not triggered. See the following issue for context - https://gitlab.com/gitlab-org/gitlab/-/issues/249524.
+      allow_failure: true
+    - when: never
   variables:
     DEPLOYMENT_ENVIRONMENT: development
   needs: