Merge branch 'make-tf-autodevops-friendly' into 'master'

Update terraform pipeline based on work on API gateway

See merge request !45

.editorconfig

+root=true
+
+[*.{yml,yaml}]
+indent_size=2
+indent_style=space
+max_line_length=120

CHANGELOG.md

@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.2.0] - 2023-06-08
+
+### Added
+
+- `terraform-pipeline.yml`: jobs may now be disabled using Auto DevOps-style
+  `..._DISABLED` variables.
+
+### Modified
+
+- `terraform-pipeline.yml`: modified job names to be more explicit that they run terraform.
+- `terraform-pipeline.yml`: no longer override Auto DevOps workflow and stage configuration.
+
 ## [2.1.0] - 2023-05-10
 
 ### Added

auto-devops-stages.yml

+# A copy of the Auto DevOps stages. Useful for standalone templates which need to import Auto DevOps-like templates.
+stages:
+  - build
+  - test
+  - deploy  # dummy stage to follow the template guidelines
+  - review
+  - dast
+  - staging
+  - canary
+  - production
+  - incremental rollout 10%
+  - incremental rollout 25%
+  - incremental rollout 50%
+  - incremental rollout 100%
+  - performance
+  - cleanup

auto-devops/terraform-pipeline.yml

@@ -7,7 +7,7 @@
 # include:
 #   - project: 'uis/devops/continuous-delivery/ci-templates'
 #     file: '/auto-devops/terraform-pipeline.yml'
-#     ref: v1.0.0
+#     ref: v2.2.0
 #
 # Variables
 #
@@ -22,56 +22,64 @@
 #   TERRAFORM_DEPLOY_VERSION - The version of the TERRAFORM_DEPLOY_IMAGE to use. Defaults to 1.4.
 #
 # Tests/linting
-# 
-# This template includes four jobs in the test stage, tf-validate, tf-fmt, tf-lint, and tf-sec. These jobs run
-# on every commit pushed or if a merge request is open. For more info see the comments alongside each of these
-# job definitions below.
-# 
+#
+# This template includes four jobs in the test stage, terraform-validate, terraform-fmt, terraform-tflint, and
+# terraform-tfsec. These jobs run on every commit pushed or if a merge request is open. For more info see the comments
+# alongside each of these job definitions below.
+#
 # Terraform plan
-# 
+#
 # When commits are pushed to an open merge request, or a commit is merged to the default branch, a Terraform plan is
-# generated for each of the development, staging, and production workspaces. See the comments alongside the .tf-plan
-# job for more information on this.
+# generated for each of the development, staging, and production workspaces. See the comments alongside the
+# .terraform-plan job for more information on this.
 #
 # Terraform apply
 #
-# The Terraform apply jobs only run when commits are merged to the default branch. The apply jobs require a
-# plan artifact from the plan job relating to the same workspace. See the comments alongside the .tf-apply job for
+# The Terraform apply jobs only run when commits are merged to the default branch. The apply jobs require a plan
+# artifact from the plan job relating to the same workspace. See the comments alongside the .terraform-apply job for
 # more information on this.
-# 
+#
 # Default workflow
 #
-# The default workflow is that as soon as a merge request is opened the test and plan stage jobs start running. Once
-# a merge request is merged to the default branch the plan jobs for all environments run again. Then, the staging
+# The default workflow is that as soon as a commit is made or merge request is opened the test stage jobs start running.
+# For merge requests, the various terraform test jobs will run for both the original commit and the post-merge commit.
+# Once a merge request is merged to the default branch the plan jobs for all environments run again. Then, the staging
 # environment apply job runs automatically (staging is therefore always an accurate representation of the default
-# branch). All other  environments require a manual trigger in the pipeline UI to start their apply jobs.
+# branch). All other environments require a manual trigger in the pipeline UI to start their apply jobs.
+#
+# Disabling jobs
+#
+# Following the AutoDevOps model, if the following variables are defined, the associated jobs will be disabled:
+#
+# - TERRAFORM_VALIDATE_DISABLED
+# - TERRAFORM_FMT_DISABLED
+# - TFLINT_DISABLED
+# - TFSEC_DISABLED
+# - TERRAFORM_PLAN_DEVELOPMENT_DISABLED
+# - TERRAFORM_APPLY_DEVELOPMENT_DISABLED
+# - TERRAFORM_PLAN_STAGING_DISABLED
+# - TERRAFORM_APPLY_STAGING_DISABLED
+# - TERRAFORM_PLAN_PRODUCTION_DISABLED
+# - TERRAFORM_APPLY_PRODUCTION_DISABLED
+#
+# If you need to add terraform jobs for any workspaces beyond "development",
+# "staging" and "production", copy or extend the terraform-{plan,apply}-... jobs at
+# the bottom of the file.
 
-# This is a workflow rule to avoid duplicate pipelines triggering when a merge request is opened/pushed to.
-workflow:
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-    - if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS"
-      when: never
-    - if: "$CI_COMMIT_BRANCH || $CI_COMMIT_TAG"
-  
 variables:
   TERRAFORM_DEPLOY_IMAGE: registry.gitlab.developers.cam.ac.uk/uis/devops/infra/dockerimages/logan-terraform
   TERRAFORM_DEPLOY_VERSION: "1.4"
   TF_DATA_DIR: ${CI_PROJECT_DIR}/terraform_data
 
-stages:
-  - test
-  - plan
-  - apply
-
 # This job generates a plan and stores it as an artifact for consumption by a related apply job. It uses the
 # -detailed-exitcode flag to enable the additional exit code 2, meaning that the plan executed successfully but
 # that changes are required (see https://developer.hashicorp.com/terraform/cli/commands/plan#detailed-exitcode).
 # Therefore, we set it to allow failure if the exit code is 2. This makes it very quick to see in the pipeline view
 # if a plan requires any changes. The plan is also stored as a terraform report artifact, meaning that the
 # merge request UI displays a summary of the plan with a link to the detailed output.
-.tf-plan:
-  stage: plan
+#
+# Plans only run when MRs have actually been opened or on the default branch.
+.terraform-plan:
   image: ${TERRAFORM_DEPLOY_IMAGE}:${TERRAFORM_DEPLOY_VERSION}
   variables:
     TF_WORKSPACE: $DEPLOYMENT_ENVIRONMENT
@@ -98,11 +106,10 @@ stages:
     reports:
       terraform: ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan.report
 
-# This job takes a plan object generated by a .tf-plan job and applies it to the relevant workspace.
+# This job takes a plan object generated by a .terraform-plan job and applies it to the relevant workspace.
 # The job only runs if the commit branch is the default branch. If the deployment environment is "staging", the job
 # will run automatically, otherwise it will require a manual trigger in the pipeline UI.
-.tf-apply:
-  stage: apply
+.terraform-apply:
   environment: ${DEPLOYMENT_ENVIRONMENT}
   image: ${TERRAFORM_DEPLOY_IMAGE}:${TERRAFORM_DEPLOY_VERSION}
   variables:
@@ -119,35 +126,41 @@ stages:
   tags:
     - $GKE_RUNNER_TAG
 
-# This job simply runs terraform validate, only if a merge request is open.
-tf-validate:
+# This job simply runs terraform validate.
+terraform-validate:
   stage: test
   image: ${TERRAFORM_DEPLOY_IMAGE}:${TERRAFORM_DEPLOY_VERSION}
   script: |
     terraform init -backend=false
     terraform validate
   rules:
+    - if: $TERRAFORM_VALIDATE_DISABLED
+      when: never
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
       when: never
-    - when: on_success
+    - if: $CI_PIPELINE_SOURCE == "push"
   tags:
     - $GKE_RUNNER_TAG
+  needs: []
 
-# This job simply runs terraform fmt, only if a merge request is open.
-tf-fmt:
+# This job simply runs terraform fmt.
+terraform-fmt:
   stage: test
   image: ${TERRAFORM_DEPLOY_IMAGE}:${TERRAFORM_DEPLOY_VERSION}
   script: terraform fmt -diff -recursive -check
   rules:
+    - if: $TERRAFORM_FMT_DISABLED
+      when: never
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
       when: never
-    - when: on_success
+    - if: $CI_PIPELINE_SOURCE == "push"
   tags:
     - $GKE_RUNNER_TAG
+  needs: []
 
 # The tflint (https://github.com/terraform-linters/tflint) tool contains many linting rules and is
 # basically the community standard. We are enabling almost all available rules.
-tf-lint:
+tflint:
   stage: test
   image:
     name: ghcr.io/terraform-linters/tflint:latest
@@ -166,19 +179,21 @@ tf-lint:
         --enable-rule terraform_required_version \
         --enable-rule terraform_typed_variables \
         --enable-rule terraform_unused_declarations \
-        --enable-rule terraform_unused_required_providers \
-        .
+        --enable-rule terraform_unused_required_providers
   rules:
+    - if: $TFLINT_DISABLED
+      when: never
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
       when: never
-    - when: on_success
+    - if: $CI_PIPELINE_SOURCE == "push"
   tags:
     - $GKE_RUNNER_TAG
+  needs: []
 
 # tfsec (https://aquasecurity.github.io/tfsec) is a comprehensive static analysis tool with many security
 # related checks for multiple cloud providers. This job uploads a junit report of the test results which is viewable
 # in the merge request UI. To ignore specific checks see - https://aquasecurity.github.io/tfsec/v1.28.1/guides/configuration/ignores/
-tf-sec:
+tfsec:
   stage: test
   image:
     name: aquasec/tfsec:latest
@@ -187,9 +202,9 @@ tf-sec:
     mkdir ${TF_DATA_DIR}
     tfsec --force-all-dirs --include-passed --format lovely,junit --out ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-tfsec
   rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $TFSEC_DISABLED
       when: never
-    - when: on_success
+    - if: $CI_PIPELINE_SOURCE == "push"
   tags:
     - $GKE_RUNNER_TAG
   allow_failure: true
@@ -198,42 +213,73 @@ tf-sec:
     when: always
     reports:
       junit: ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-tfsec.junit
+  needs: []
 
 # The following jobs make up the default workflow for our standard three-environment products. For products with
 # more/different environments, these jobs can be overridden or extended as require in the product's infrastructure
 # repository directly.
-tf-plan-development:
-  extends: .tf-plan
+terraform-plan-development:
+  extends: .terraform-plan
+  stage: review
+  rules:
+    - if: $TERRAFORM_PLAN_DEVELOPMENT_DISABLED
+      when: never
+    - !reference [.terraform-plan, rules]
   variables:
     DEPLOYMENT_ENVIRONMENT: development
 
-tf-apply-development:
-  extends: .tf-apply
+terraform-apply-development:
+  extends: .terraform-apply
+  stage: review
+  rules:
+    - if: $TERRAFORM_APPLY_DEVELOPMENT_DISABLED
+      when: never
+    - !reference [.terraform-apply, rules]
   variables:
     DEPLOYMENT_ENVIRONMENT: development
   needs:
-    - tf-plan-development
+    - terraform-plan-development
 
-tf-plan-staging:
-  extends: .tf-plan
+terraform-plan-staging:
+  extends: .terraform-plan
+  stage: staging
+  rules:
+    - if: $TERRAFORM_PLAN_STAGING_DISABLED
+      when: never
+    - !reference [.terraform-plan, rules]
   variables:
     DEPLOYMENT_ENVIRONMENT: staging
-  
-tf-apply-staging:
-  extends: .tf-apply
+
+terraform-apply-staging:
+  extends: .terraform-apply
+  stage: staging
+  rules:
+    - if: $TERRAFORM_APPLY_STAGING_DISABLED
+      when: never
+    - !reference [.terraform-apply, rules]
   variables:
     DEPLOYMENT_ENVIRONMENT: staging
   needs:
-    - tf-plan-staging
+    - terraform-plan-staging
 
-tf-plan-production:
-  extends: .tf-plan
+terraform-plan-production:
+  extends: .terraform-plan
+  stage: production
+  rules:
+    - if: $TERRAFORM_PLAN_PRODUCTION_DISABLED
+      when: never
+    - !reference [.terraform-plan, rules]
   variables:
     DEPLOYMENT_ENVIRONMENT: production
-  
-tf-apply-production:
-  extends: .tf-apply
+
+terraform-apply-production:
+  extends: .terraform-apply
+  stage: production
+  rules:
+    - if: $TERRAFORM_APPLY_PRODUCTION_DISABLED
+      when: never
+    - !reference [.terraform-apply, rules]
   variables:
     DEPLOYMENT_ENVIRONMENT: production
   needs:
-    - tf-plan-production
+    - terraform-plan-production

terraform-pipeline.yml

+# Stand alone version of auto-devops/terraform-pipeline.yml.
+#
+# This template can be used for pure terraform projects which do not benefit from the full Auto DevOps pipeline.
+
+include:
+  - local: '/auto-devops-stages.yml'
+  - local: '/auto-devops/terraform-pipeline.yml'