FAQ | This is a LIVE service | Changelog

Skip to content
Commits on Source (4)
Hide whitespace changes
Inline Side-by-side
CHANGELOG.md
View file @ 36d951f8
......@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [4.0.0] - 2024-04-05
### Changed
- BREAKING CHANGE: removed legacy `/auto-devops/terraform-deployment.yml` and
`/auto-devops/terraform-lint.yml` templates as these should not be being used
any more.
- Refactored `/auto-devops/terraform-pipeline.yml` template so that the hidden
"template" jobs are now in their own `/terraform-pipeline-base.yml` template
file. This allows more flexibility for some projects which do not/cannot
follow the standard three-environment deployment defined in the
`/auto-devops/terraform-pipeline.yml` template.
- Refactored the Terraform testing/linting jobs into their own
`/terraform-lint.yml` template. This is useful for projects which are not
being deployed via CI/CD but which we do want to test in a pipeline.
## [3.8.2] - 2024-04-10
### Fixed
......
auto-devops/terraform-deployment.yml deleted 100644 → 0
View file @ e50faf87
# Job for running a Terraform deployment flow from staging to production
#
# This template is intended to be "include"-d from CI configurations. An
# example of how to include this template:
#
# include:
# - project: 'uis/devops/continuous-delivery/ci-templates'
# file: '/auto-devops/terraform-deployment.yml'
# ref: v1.7.2
#
# The default docker terraform image can be overridden by specifying
# `TERRAFORM_DEPLOY_IMAGE` and/or `TERRAFORM_DEPLOY_VERSION` (typically just the
# latter). This should be the image from .logan.yaml in the project deploy repo.
# Terraform typically requires project folder admin level credentials, this must
# be available as the GitLab project variable `PRODUCT_ADMIN_CREDENTIALS`
#
# Deployment requires the following stages:
# stages:
# - terraform-lint
# - terraform-plan-staging
# - terraform-apply-staging
# - terraform-plan-production
# - terraform-apply-production
variables:
GOOGLE_CREDENTIALS: ${PRODUCT_ADMIN_CREDENTIALS}
GOOGLE_APPLICATION_CREDENTIALS: "/tmp/google_application_credentials.json"
TF_DATA_DIR: "${CI_PROJECT_DIR}/terraform_data"
.terraform:
environment: ${DEPLOYMENT_ENVIRONMENT}/${CI_COMMIT_REF_SLUG}
image: ${TERRAFORM_DEPLOY_IMAGE}:${TERRAFORM_DEPLOY_VERSION}
services:
- docker:dind
variables:
# The docker daemon is available with a network connection instead of
# the default /var/run/docker.sock socket, hence the following.
DOCKER_HOST: "tcp://docker:2375"
TERRAFORM_DEPLOY_IMAGE: "registry.gitlab.developers.cam.ac.uk/uis/devops/infra/dockerimages/logan-terraform"
TERRAFORM_DEPLOY_VERSION: "0.14"
artifacts:
# NOTE: the terraform plan file may contain sensitive data, keep the lifetime
# short to reduce the chances of it ending up in a backup. TODO; Encrypt it?
name: "terraform-data-$CI_COMMIT_REF_NAME-$DEPLOYMENT_ENVIRONMENT"
paths:
- ${CI_PROJECT_DIR}/terraform_data/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}-tfplan
expire_in: 2 hours
script:
- echo $GOOGLE_CREDENTIALS > /tmp/google_application_credentials.json
- echo "Deployment environment tier ${DEPLOYMENT_ENVIRONMENT}"
- terraform version
- TF_WORKSPACE="${DEPLOYMENT_ENVIRONMENT}"
- terraform init -input=false
- terraform workspace select ${DEPLOYMENT_ENVIRONMENT}
.terraform-plan:
extends: .terraform
before_script:
# clear any residual terraform data.
- rm -vrf .terraform/* ${TF_DATA_DIR}/* || true
- !reference [.terraform, script]
script:
- terraform plan -out=${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}-tfplan -input=false
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($DEPLOYMENT_ENVIRONMENT == "staging" || $DEPLOYMENT_ENVIRONMENT == "production")'
.terraform-apply:
extends: .terraform
before_script:
- !reference [.terraform, script]
script:
- terraform apply -auto-approve ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}-tfplan
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $DEPLOYMENT_ENVIRONMENT == "production"'
when: manual
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $DEPLOYMENT_ENVIRONMENT == "staging"'
lint:
extends: .terraform
stage: terraform-lint
before_script:
- !reference [.terraform, script]
script:
- terraform fmt -diff -recursive -check
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
plan-staging:
extends: .terraform-plan
stage: terraform-plan-staging
variables:
DEPLOYMENT_ENVIRONMENT: "staging"
apply-staging:
extends: .terraform-apply
stage: terraform-apply-staging
variables:
DEPLOYMENT_ENVIRONMENT: "staging"
needs:
- job: plan-staging
artifacts: true
plan-production:
extends: .terraform-plan
stage: terraform-plan-production
variables:
DEPLOYMENT_ENVIRONMENT: "production"
needs:
- job: apply-staging
apply-production:
extends: .terraform-apply
stage: terraform-apply-production
variables:
DEPLOYMENT_ENVIRONMENT: "production"
needs:
- job: plan-production
artifacts: true
auto-devops/terraform-lint.yml deleted 100644 → 0
View file @ e50faf87
# Job for linting Terraform configuration
#
# This template is intended to be "include"-d from CI configurations. An
# example of how to include this template:
#
# include:
# - project: 'uis/devops/continuous-delivery/ci-templates'
# file: '/auto-devops/terraform-lint.yml'
# ref: v1.6.0
#
# The default docker terraform image can be overridden by specifying
# `TERRAFORM_LINT_IMAGE` and/or `TERRAFORM_LINT_VERSION` (typically just the
# latter) variables for the `lint_terraform` task.
#
# For example,
#
# lint_terraform:
# variables:
# TERRAFORM_LINT_VERSION: "1.1.0"
lint_terraform:
stage: test
image: docker:git
services:
- docker:dind
script:
# Run terraform's format checker.
- docker run --rm -v "$(pwd):/tmp/tf" "${TERRAFORM_LINT_IMAGE}:${TERRAFORM_LINT_VERSION}" fmt -diff -recursive -check /tmp/tf
rules:
- if: '$TEST_DISABLED'
when: never
- if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
when: on_success
- when: never
variables:
TERRAFORM_LINT_IMAGE: "hashicorp/terraform"
TERRAFORM_LINT_VERSION: "1.0.4"
auto-devops/terraform-pipeline.yml
View file @ 36d951f8
# Implements the UIS DevOps workflow for deploying product Terraform configurations.
#
# This template is intended to be generic and to be "include"-d in .gitlab-ci.yml configurations.
# This template is intended to be "include"-d in .gitlab-ci.yml configurations. It uses the base jobs defined in
# terraform-pipeline-base.yml to build a drop in deployment pipeline for our standard GCP products.
#
# To use this template, make sure it's the first template to be "include"-d.
#
# include:
# - project: 'uis/devops/continuous-delivery/ci-templates'
# file: '/auto-devops/terraform-pipeline.yml'
# ref: v2.2.0
# ref: vx.x.x
#
# Variables
#
......@@ -19,42 +20,26 @@
# TERRAFORM_DEPLOY_IMAGE - The image to use in the CI jobs. Defaults to
# registry.gitlab.developers.cam.ac.uk/uis/devops/infra/dockerimages/logan-terraform
#
# TERRAFORM_DEPLOY_VERSION - The version of the TERRAFORM_DEPLOY_IMAGE to use. Defaults to 1.4.
# TERRAFORM_DEPLOY_VERSION - The version of the TERRAFORM_DEPLOY_IMAGE to use.
#
# Tests/linting
#
# This template includes four jobs in the test stage, terraform-validate, terraform-fmt, tflint, and trivy.
# These jobs run on every commit pushed or if a merge request is open. For more info see the comments
# alongside each of these job definitions below.
#
# Terraform plan
#
# When commits are pushed to an open merge request, or a commit is merged to the default branch, a Terraform plan is
# generated for each of the development, staging, and production workspaces. See the comments alongside the
# .terraform-plan job for more information on this.
#
# Terraform apply
#
# The Terraform apply jobs only run when commits are merged to the default branch. The apply jobs require a plan
# artifact from the plan job relating to the same workspace. See the comments alongside the .terraform-apply job for
# more information on this.
# This template includes the terraform-lint.yml template which adds multiple testing and linting jobs to the pipeline.
# For more information on how these jobs work and how to disable them if required see the terraform-lint.yml file.
#
# Default workflow
#
# The default workflow is that as soon as a commit is made or merge request is opened the test stage jobs start running.
# For merge requests, the various terraform test jobs will run for both the original commit and the post-merge commit.
# Once a merge request is merged to the default branch the plan jobs for all environments run again. Then, the staging
# environment apply job runs automatically (staging is therefore always an accurate representation of the default
# branch). All other environments require a manual trigger in the pipeline UI to start their apply jobs.
# The default workflow is that as soon as a commit is made or a merge request is opened the test stage jobs start
# running. For merge requests, the various terraform test jobs will run for both the original commit and the post-merge
# commit and a terraform plan job will run for each environment. Once a merge request is merged to the default branch
# the plan jobs for all environments run again. Then, the staging environment apply job runs automatically (staging is
# therefore always an accurate representation of the default branch). All other environments require a manual trigger in
# the pipeline UI to start their apply jobs.
#
# Disabling jobs
#
# Following the AutoDevOps model, if the following variables are defined, the associated jobs will be disabled:
#
# - TERRAFORM_VALIDATE_DISABLED
# - TERRAFORM_FMT_DISABLED
# - TFLINT_DISABLED
# - TRIVY_DISABLED
# - TERRAFORM_PLAN_DEVELOPMENT_DISABLED
# - TERRAFORM_APPLY_DEVELOPMENT_DISABLED
# - TERRAFORM_PLAN_STAGING_DISABLED
......@@ -62,214 +47,17 @@
# - TERRAFORM_PLAN_PRODUCTION_DISABLED
# - TERRAFORM_APPLY_PRODUCTION_DISABLED
#
# If you need to add terraform jobs for any workspaces beyond "development",
# "staging" and "production", copy or extend the terraform-{plan,apply}-... jobs at
# the bottom of the file.
variables:
TERRAFORM_DEPLOY_IMAGE: registry.gitlab.developers.cam.ac.uk/uis/devops/infra/dockerimages/logan-terraform
TERRAFORM_DEPLOY_VERSION: "1.4"
TF_DATA_DIR: ${CI_PROJECT_DIR}/terraform_data
# Rules which only run test jobs on commit pipelines if there is not an active MR. This avoids duplicate jobs being
# created.
.test-job-rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS"
when: never
- if: "$CI_COMMIT_BRANCH || $CI_COMMIT_TAG"
# These variables are required when using docker-in-docker with TLS enabled via the Kubernetes executor for GitLab
# runner. For more information see
# https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#docker-in-docker-with-tls-enabled-in-kubernetes.
.docker-in-docker:
services:
- docker:24-dind
variables:
DOCKER_HOST: tcp://docker:2376
DOCKER_TLS_CERTDIR: "/certs"
DOCKER_TLS_VERIFY: 1
DOCKER_CERT_PATH: "/certs/client"
# This job generates a plan and stores it as an artifact for consumption by a related apply job. It uses the
# -detailed-exitcode flag to enable the additional exit code 2, meaning that the plan executed successfully but
# that changes are required (see https://developer.hashicorp.com/terraform/cli/commands/plan#detailed-exitcode).
# Therefore, we set it to allow failure if the exit code is 2. This makes it very quick to see in the pipeline view
# if a plan requires any changes. The plan is also stored as a terraform report artifact, meaning that the
# merge request UI displays a summary of the plan with a link to the detailed output.
#
# Plans only run when MRs have actually been opened or on the default branch.
.terraform-plan:
extends: .docker-in-docker
image: ${TERRAFORM_DEPLOY_IMAGE}:${TERRAFORM_DEPLOY_VERSION}
variables:
TF_WORKSPACE: $DEPLOYMENT_ENVIRONMENT
environment:
name: $DEPLOYMENT_ENVIRONMENT
action: prepare
resource_group: $DEPLOYMENT_ENVIRONMENT
script: |
unset GOOGLE_APPLICATION_CREDENTIALS
for target in $TF_PLAN_TARGET; do
TF_PLAN_ARGS="$TF_PLAN_ARGS --target $target"
done
terraform init
terraform plan $TF_PLAN_ARGS -out=${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan -detailed-exitcode || exit_code=$?
terraform show -json ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan > ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan.json
cat ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan.json | jq -r '([.resource_changes[]?.change.actions?]|flatten)|{"create":(map(select(.=="create"))|length),"update":(map(select(.=="update"))|length),"delete":(map(select(.=="delete"))|length)}' > ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan.report
exit $exit_code
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_TAG
- when: never
tags:
- $GKE_RUNNER_TAG
allow_failure:
exit_codes: 2 # See https://www.terraform.io/cli/commands/plan#detailed-exitcode
artifacts:
expire_in: 2 days
when: always
paths:
- ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan
reports:
terraform: ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan.report
needs:
- job: terraform-validate
optional: true
- job: terraform-fmt
optional: true
- job: tflint
optional: true
- job: trivy
optional: true
# Extending this pipeline
#
# If you need to add terraform jobs for any workspaces beyond "development", "staging" and "production", you can copy
# an existing environment's terraform-{plan,apply}-... jobs, changing the references to the existing workspace name.
# This job takes a plan object generated by a .terraform-plan job and applies it to the relevant workspace.
# The job only runs if the commit branch is the default branch. If the deployment environment is "staging", the job
# will run automatically, otherwise it will require a manual trigger in the pipeline UI.
.terraform-apply:
extends: .docker-in-docker
image: ${TERRAFORM_DEPLOY_IMAGE}:${TERRAFORM_DEPLOY_VERSION}
variables:
TF_WORKSPACE: $DEPLOYMENT_ENVIRONMENT
environment:
name: $DEPLOYMENT_ENVIRONMENT
action: start
resource_group: $DEPLOYMENT_ENVIRONMENT
script: |
unset GOOGLE_APPLICATION_CREDENTIALS
terraform init
terraform apply ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan
rules:
- if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_TAG) && $DEPLOYMENT_ENVIRONMENT == "staging"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_TAG
when: manual
- when: never
tags:
- $GKE_RUNNER_TAG
# This job simply runs terraform validate.
terraform-validate:
stage: test
image: ${TERRAFORM_DEPLOY_IMAGE}:${TERRAFORM_DEPLOY_VERSION}
script: |
terraform init -backend=false
terraform validate
rules:
- if: $TERRAFORM_VALIDATE_DISABLED
when: never
- !reference [.test-job-rules]
tags:
- $GKE_RUNNER_TAG
needs: []
# This job simply runs terraform fmt.
terraform-fmt:
stage: test
image: ${TERRAFORM_DEPLOY_IMAGE}:${TERRAFORM_DEPLOY_VERSION}
script: terraform fmt -diff -recursive -check
rules:
- if: $TERRAFORM_FMT_DISABLED
when: never
- !reference [.test-job-rules]
tags:
- $GKE_RUNNER_TAG
needs: []
# The tflint (https://github.com/terraform-linters/tflint) tool contains many linting rules and is
# basically the community standard. We are enabling almost all available rules.
tflint:
stage: test
image:
name: ghcr.io/terraform-linters/tflint:latest
entrypoint: [""]
before_script: |
if [ ! -f .tflint.hcl ]; then
cat > .tflint.hcl <<EOL
plugin "terraform" {
enabled = true
preset = "all"
}
EOL
fi
script: |
tflint --init
tflint --recursive --disable-rule=terraform_standard_module_structure
rules:
- if: $TFLINT_DISABLED
when: never
- !reference [.test-job-rules]
tags:
- $GKE_RUNNER_TAG
needs: []
# Trivy (https://aquasecurity.github.io/trivy) is a comprehensive and versatile security scanner.
# Trivy has scanners that look for security issues, and targets where it can find those issues.
# This job uploads a junit report of the test results which is viewable in the merge request UI.
# To ignore specific checks see -
# https://aquasecurity.github.io/trivy/v0.50/docs/configuration/filtering/#trivyignoreyaml.
# By default, we're ignoring certain checks which we have decided to allow due to our boilerplate template design.
trivy:
stage: test
image:
name: aquasec/trivy:latest
entrypoint: [""]
variables:
TRIVY_IGNORE: |-
AVD-GCP-0066
AVD-GCP-0029
AVD-GCP-0014
AVD-GCP-0017
AVD-GCP-0025
AVD-GCP-0016
AVD-GCP-0022
AVD-GCP-0020
before_script: |
mkdir ${TF_DATA_DIR}
for i in $TRIVY_IGNORE; do echo $i >> .trivyignore; done
script: |
trivy fs --scanners misconfig --format json --output trivy.json --exit-code 1 .
after_script: |
trivy convert --format template --template "@/contrib/junit.tpl" --output ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-trivy.junit trivy.json
trivy convert --format table --output trivy.out trivy.json && cat trivy.out
rules:
- if: $TRIVY_DISABLED
when: never
- !reference [.test-job-rules]
tags:
- $GKE_RUNNER_TAG
allow_failure: true
artifacts:
expire_in: 1 week
when: always
reports:
junit: ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-trivy.junit
needs: []
include:
- local: '/terraform-pipeline-base.yml'
- local: '/terraform-lint.yml'
# The following jobs make up the default workflow for our standard three-environment products. For products with
# more/different environments, these jobs can be overridden or extended as require in the product's infrastructure
# repository directly.
# additional environments, these jobs can be copied as require in the product's infrastructure repository directly.
terraform-plan-development:
extends: .terraform-plan
stage: review
......
fragments/docker-in-docker.yml 0 → 100644
View file @ 36d951f8
# These variables are required when using docker-in-docker with TLS enabled via the Kubernetes executor for GitLab
# runner. For more information see
# https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#docker-in-docker-with-tls-enabled-in-kubernetes.
.docker-in-docker:
services:
- docker:24-dind
variables:
DOCKER_HOST: tcp://docker:2376
DOCKER_TLS_CERTDIR: "/certs"
DOCKER_TLS_VERIFY: 1
DOCKER_CERT_PATH: "/certs/client"
terraform-lint.yml 0 → 100644
View file @ 36d951f8
# This template implements a generic set of terraform-related testing/linting jobs. It is usually included in product
# pipelines via the /auto-devops/terraform-pipeline.yml template. However, in some situations it might be necessary to
# include this template directly, for example if a specific repository is not suitable for CI pipeline deployments but
# you still wish to test/lint the Terraform code.
#
# This template includes four jobs in the test stage, terraform-validate, terraform-fmt, tflint, and trivy. By default,
# the jobs run on every commit pushed or if a merge request is open. However, this can be changed by overriding
# .test-job-rules in your local .gitlab-ci.yml file if required. For more info see the comments alongside each of these
# job definitions below.
#
# Disabling jobs
#
# Following the AutoDevOps model, if the following variables are defined, the associated jobs will be disabled:
#
# - TERRAFORM_VALIDATE_DISABLED
# - TERRAFORM_FMT_DISABLED
# - TFLINT_DISABLED
# - TRIVY_DISABLED
# Rules which only run test jobs on commit pipelines if there is not an active MR. This avoids duplicate jobs being
# created.
.test-job-rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS"
when: never
- if: "$CI_COMMIT_BRANCH || $CI_COMMIT_TAG"
terraform-validate:
stage: test
image: ${TERRAFORM_DEPLOY_IMAGE}:${TERRAFORM_DEPLOY_VERSION}
script: |
terraform init -backend=false
terraform validate
rules:
- if: $TERRAFORM_VALIDATE_DISABLED
when: never
- !reference [.test-job-rules]
tags:
- $GKE_RUNNER_TAG
needs: []
terraform-fmt:
stage: test
image: ${TERRAFORM_DEPLOY_IMAGE}:${TERRAFORM_DEPLOY_VERSION}
script: terraform fmt -diff -recursive -check
rules:
- if: $TERRAFORM_FMT_DISABLED
when: never
- !reference [.test-job-rules]
tags:
- $GKE_RUNNER_TAG
needs: []
# The tflint (https://github.com/terraform-linters/tflint) tool contains many linting rules and is basically the
# community standard. We are enabling almost all available rules.
tflint:
stage: test
image:
name: ghcr.io/terraform-linters/tflint:latest
entrypoint: [""]
before_script: |
if [ ! -f .tflint.hcl ]; then
cat > .tflint.hcl <<EOL
plugin "terraform" {
enabled = true
preset = "all"
}
EOL
fi
script: |
tflint --init
tflint --recursive --disable-rule=terraform_standard_module_structure
rules:
- if: $TFLINT_DISABLED
when: never
- !reference [.test-job-rules]
tags:
- $GKE_RUNNER_TAG
needs: []
# Trivy (https://aquasecurity.github.io/trivy) is a comprehensive and versatile security scanner. Trivy has scanners
# that look for security issues, and targets where it can find those issues. This job uploads a junit report of the test
# results which is viewable in the merge request UI. To ignore specific checks see -
# https://aquasecurity.github.io/trivy/v0.50/docs/configuration/filtering/#trivyignoreyaml. By default, we're ignoring
# certain checks which we have decided to allow due to our boilerplate template design.
trivy:
stage: test
image:
name: aquasec/trivy:latest
entrypoint: [""]
variables:
TRIVY_IGNORE: |-
AVD-GCP-0066
AVD-GCP-0029
AVD-GCP-0014
AVD-GCP-0017
AVD-GCP-0025
AVD-GCP-0016
AVD-GCP-0022
AVD-GCP-0020
before_script: |
mkdir ${TF_DATA_DIR}
for i in $TRIVY_IGNORE; do echo $i >> .trivyignore; done
script: |
trivy fs --scanners misconfig --format json --output trivy.json --exit-code 1 .
after_script: |
trivy convert --format template --template "@/contrib/junit.tpl" --output ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-trivy.junit trivy.json
trivy convert --format table --output trivy.out trivy.json && cat trivy.out
rules:
- if: $TRIVY_DISABLED
when: never
- !reference [.test-job-rules]
tags:
- $GKE_RUNNER_TAG
allow_failure: true
artifacts:
expire_in: 1 week
when: always
reports:
junit: ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-trivy.junit
needs: []
terraform-pipeline-base.yml 0 → 100644
View file @ 36d951f8
# This template implements generic template jobs for use when deploying terraform configurations.
#
# This template is not usually used directly. Instead, most products will want to use the more opinionated
# /auto-devops/terraform-pipeline.yml template which builds a full deployment pipeline using these "base" jobs. This
# template should only be directly "include"-d in a .gitlab-ci.yml file when a product cannot use the
# /auto-devops/terraform-pipeline.yml template for some reason.
#
# Variables
#
# The following variables are used to configure job behavior.
#
# GKE_RUNNER_TAG - Use this to specify the required tag for the product's runner (see
# https://gitlab.developers.cam.ac.uk/uis/devops/devhub/gitlab-runner-infrastructure)
#
# TERRAFORM_DEPLOY_IMAGE - The image to use in the CI jobs. Defaults to
# registry.gitlab.developers.cam.ac.uk/uis/devops/infra/dockerimages/logan-terraform
#
# TERRAFORM_DEPLOY_VERSION - The version of the TERRAFORM_DEPLOY_IMAGE to use.
include:
- local: /fragments/docker-in-docker.yml
variables:
TERRAFORM_DEPLOY_IMAGE: registry.gitlab.developers.cam.ac.uk/uis/devops/infra/dockerimages/logan-terraform
TERRAFORM_DEPLOY_VERSION: "1.4"
TF_DATA_DIR: ${CI_PROJECT_DIR}/terraform_data
# This job generates a plan and stores it as an artifact for consumption by a related apply job. It uses the
# -detailed-exitcode flag to enable the additional exit code 2, meaning that the plan executed successfully but
# that changes are required (see https://developer.hashicorp.com/terraform/cli/commands/plan#detailed-exitcode).
# Therefore, we set it to allow failure if the exit code is 2. This makes it very quick to see in the pipeline view
# if a plan requires any changes. The plan is also stored as a terraform report artifact, meaning that the
# merge request UI displays a summary of the plan with a link to the detailed output.
#
# Plans only run when MRs have actually been opened or on the default branch.
.terraform-plan:
extends: .docker-in-docker
image: ${TERRAFORM_DEPLOY_IMAGE}:${TERRAFORM_DEPLOY_VERSION}
variables:
TF_WORKSPACE: $DEPLOYMENT_ENVIRONMENT
environment:
name: $DEPLOYMENT_ENVIRONMENT
action: prepare
resource_group: $DEPLOYMENT_ENVIRONMENT
script: |
# For historical reasons the logan-terraform image sets GOOGLE_APPLICATION_CREDENTIALS=/credentials.json. We no not
# use a credentials file so we need to unset this variable. In the future this will be fixed in the logan-terraform
# image and this statement will be able to be removed.
unset GOOGLE_APPLICATION_CREDENTIALS
for target in $TF_PLAN_TARGET; do
TF_PLAN_ARGS="$TF_PLAN_ARGS --target $target"
done
terraform init
# When using --detailed-exitcode the plan command will exit with one of the following.
#
# - 0: An exit code of 0 means the command completed successfully and there were no planned changes required.
# - 1: An exit code of 1 means that the command did not complete successfully.
# - 2: An exit code of 2 means that the command completed successfully but there were proposed changes in the resulting plan.
#
# By allowing this job to exit with exit code 2 we are able to show an amber warning icon in the GitLab UI. This
# makes it easy to spot when an environment requires changes following a plan job.
(terraform plan $TF_PLAN_ARGS -out=${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan -detailed-exitcode && exit_code=$?) || exit_code=$?
if [ "$exit_code" -eq 1 ]; then exit 1; fi
if [ -f "${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan" ]; then
terraform show -json ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan > ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan.json
cat ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan.json | jq -r '([.resource_changes[]?.change.actions?]|flatten)|{"create":(map(select(.=="create"))|length),"update":(map(select(.=="update"))|length),"delete":(map(select(.=="delete"))|length)}' > ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan.report
fi
# If terraform plan exited with 2 then we need to override the last exit code.
if [ "$exit_code" -eq 2 ]; then exit 2; fi
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_TAG
- when: never
tags:
- $GKE_RUNNER_TAG
allow_failure:
exit_codes: 2 # See https://www.terraform.io/cli/commands/plan#detailed-exitcode
artifacts:
expire_in: 2 days
when: always
paths:
- ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan
reports:
terraform: ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan.report
needs:
- job: terraform-validate
optional: true
- job: terraform-fmt
optional: true
- job: tflint
optional: true
- job: trivy
optional: true
# This job takes a plan object generated by a .terraform-plan job and applies it to the relevant workspace.
# The job only runs if the commit branch is the default branch. If the deployment environment is "staging", the job
# will run automatically, otherwise it will require a manual trigger in the pipeline UI.
.terraform-apply:
extends: .docker-in-docker
image: ${TERRAFORM_DEPLOY_IMAGE}:${TERRAFORM_DEPLOY_VERSION}
variables:
TF_WORKSPACE: $DEPLOYMENT_ENVIRONMENT
environment:
name: $DEPLOYMENT_ENVIRONMENT
action: start
resource_group: $DEPLOYMENT_ENVIRONMENT
script: |
unset GOOGLE_APPLICATION_CREDENTIALS
terraform init
terraform apply ${TF_DATA_DIR}/${CI_COMMIT_REF_SLUG}-${DEPLOYMENT_ENVIRONMENT}.tfplan
rules:
- if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_TAG) && $DEPLOYMENT_ENVIRONMENT == "staging"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_TAG
when: manual
- when: never
tags:
- $GKE_RUNNER_TAG