From 929894f5209d90ce0439e80f321f980091f449a8 Mon Sep 17 00:00:00 2001 From: Rich Wareham <rjw57@cam.ac.uk> Date: Thu, 28 Nov 2024 10:34:29 +0000 Subject: [PATCH] feat: add the bones of an OpenAPI client generation mechanism Add the start of a re-usable OpenAPI client generation mechanism. Add two files: one which represents the repository-local customisation and one which is the re-usable template. Add the start of some documentation for the template with an eye to moving it to the guidebook. The general idea is that getting OpenAPI clients *should* be as simple as including an `openapi.yaml` file in the root of the repository. (Note the name of the file was chosen because GitLab will automatically render files so-named with swagger UI.) For our specific use case we want to generate the schema dynamically and so allow for this by means of customising an "openapi:schema" job whose role is to ensure the schema is available to downstream jobs as a CI job artefact. The default behaviour is simply to copy the schema from the repository but we override it to use the DRF spectactular Django management command. We use the `--fail-if-warn` flag to allow this job to additionally act as a "test" test that schema generation succeeds without warning. Add in the plumbing to run the [openapi-generator CLI][1] but only use it for the moment as an additional "test" job which validates the spec. [1]: https://openapi-generator.tech/ The template has been written with an eye to adding to the common pipeline and so that there can be zero configuration required to make use of it. As such the template follows the rules of the common pipeline in being "safe" to include with no side-effects unless it's triggered and all CI variables and artefacts are appropriately namespaced. Closes #24 --- .gitlab-ci.yml | 4 + .gitlab/openapi-generator-local.gitlab-ci.yml | 17 +++ .gitlab/openapi-generator.gitlab-ci.yml | 100 ++++++++++++++++++ .gitlab/openapi-generator.md | 54 ++++++++++ 4 files changed, 175 insertions(+) create mode 100644 .gitlab/openapi-generator-local.gitlab-ci.yml create mode 100644 .gitlab/openapi-generator.gitlab-ci.yml create mode 100644 .gitlab/openapi-generator.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ca9524..dfdd84b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,2 +1,6 @@ include: - local: "/.gitlab/webapp.gitlab-ci.yml" + # OpenAPI client generation templates. Destined for the common ci-templates repo. + - local: "/.gitlab/openapi-generator.gitlab-ci.yml" + # Local customisations for OpenAPI generation. + - local: "/.gitlab/openapi-generator-local.gitlab-ci.yml" diff --git a/.gitlab/openapi-generator-local.gitlab-ci.yml b/.gitlab/openapi-generator-local.gitlab-ci.yml new file mode 100644 index 0000000..aaef0b1 --- /dev/null +++ b/.gitlab/openapi-generator-local.gitlab-ci.yml @@ -0,0 +1,17 @@ +# *REPOSITORY LOCAL* specialisation of the OpenAPI generator templates. + +variables: + # We need to explicitly enable the OpenAPI generation since we don't have an openapi.yaml file in + # the root of our repository which would otherwise signal that. + OPENAPI_GENERATOR_ENABLED: "1" + +# We do not ship the OpenAPI schema in the repository and so the openapi:schema job needs to be +# extended to create the schema. +openapi:schema: + services: + - docker:dind + image: + name: docker:latest + script: + - cp secrets.env.in secrets.env + - docker compose run --rm manage spectacular --fail-on-warn --file "$OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH" diff --git a/.gitlab/openapi-generator.gitlab-ci.yml b/.gitlab/openapi-generator.gitlab-ci.yml new file mode 100644 index 0000000..5dc90f7 --- /dev/null +++ b/.gitlab/openapi-generator.gitlab-ci.yml @@ -0,0 +1,100 @@ +# Reusable template for generating API clients from an OpenAPI schema. +# +# See the openapi-generator.md file in this directory for detailed usage information. +# +# This template is intended to be included in the common pipeline and so follows the common pipeline +# rules: +# +# - No jobs are added unless triggered explicitly or via the presence of a special file. +# - All jobs can be disabled if incorrectly triggered. +# - All "public" CI variables are namespaced to avoid collisions. +# - All build artefacts are added to a configurable directory to allow customisation if that +# directory conflicts with one in the repository. +# +# This template requires the Auto DevOps stages. + +variables: + # Location of the OpenAPI schema file within the repository. + OPENAPI_GENERATOR_SCHEMA_PATH: "openapi.yaml" + + # Location of various job artefacts. These will always be relative to $CI_PROJECT_DIR. + OPENAPI_GENERATOR_ARTIFACT_DIR: "openapi" + OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH: $OPENAPI_GENERATOR_ARTIFACT_DIR/schema.yml + OPENAPI_GENERATOR_SCHEMA_VERSION_ARTIFACT: $OPENAPI_GENERATOR_ARTIFACT_DIR/schema-version + + # Specify Docker images used by jobs. + OPENAPI_GENERATOR_YQ_IMAGE: mikefarah/yq:4 + OPENAPI_GENERATOR_OPENAPI_GENERATOR_IMAGE: openapitools/openapi-generator-cli:v7.10.0 + +# Ensure that the schema is present as the $OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH artefact. The +# default behaviour is to copy the file specified by $OPENAPI_GENERATOR_SCHEMA_PATH but the job can +# be extended for more complex behaviour or dynamically generated schema. +openapi:schema: + stage: build + needs: [] + before_script: + # Ensure that the directory which is to contain the schema exists. + - mkdir -p $(dirname "$OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH") + script: + # The default behaviour is to copy the schema file to the artifact directory. + - cp "$OPENAPI_GENERATOR_SCHEMA_PATH" "$OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH" + artifacts: + paths: + - $OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH + # When extending this job, rules which *disable* jobs should be included *above* these rules. + # Rules which *enable* the job should be included *below* them. + rules: + # Never run any OpenAPI generate jobs if OPENAPI_GENERATOR_DISABLED is set. + - if: $OPENAPI_GENERATOR_DISABLED + when: never + # If no API specification is provided, never run. + - if: ($OPENAPI_GENERATOR_SCHEMA_PATH == null) || ($OPENAPI_GENERATOR_SCHEMA_PATH == "") + when: never + # If an OpenAPI spec is present in the repository, use it. + - if: $OPENAPI_GENERATOR_SCHEMA_PATH + exists: + paths: + - $OPENAPI_GENERATOR_SCHEMA_PATH + # Finally, allow OpenAPI generation to be explicitly enabled if desired. + - if: $OPENAPI_GENERATOR_ENABLED + +# Extract the version from the OpenAPI schema. This is required by some generator jobs as some +# generator templates do not respect the version set in the schema. The version is written to +# $OPENAPI_GENERATOR_SCHEMA_VERSION_ARTIFACT. Note that even generators which support reading the +# version from the schema should respect the artefact generated by this job since it provides a +# customisation point to override the package version if needed. +openapi:schema:version: + stage: build + needs: ["openapi:schema"] + image: + name: $OPENAPI_GENERATOR_YQ_IMAGE + entrypoint: [""] + script: + - yq -r ".info.version" "$OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH" >"$OPENAPI_GENERATOR_SCHEMA_VERSION_ARTIFACT" + artifacts: + paths: + - $OPENAPI_GENERATOR_SCHEMA_VERSION_ARTIFACT + rules: + - !reference ["openapi:schema", rules] + +# Template CI job for running openapi-generator. +.openapi:generator-cli: + image: + name: $OPENAPI_GENERATOR_OPENAPI_GENERATOR_IMAGE + entrypoint: [""] + variables: + # Customise how openapi-generator-cli is run. + OPENAPI_GENERATOR_CMD: "/usr/local/bin/docker-entrypoint.sh" + rules: + - !reference ["openapi:schema", rules] + +# Validate the OpenAPI schema. +openapi:schema:validate: + extends: ".openapi:generator-cli" + stage: test + needs: ["openapi:schema"] + script: + - |- + $OPENAPI_GENERATOR_CMD validate \ + --input-spec "$OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH" \ + --recommend diff --git a/.gitlab/openapi-generator.md b/.gitlab/openapi-generator.md new file mode 100644 index 0000000..25a6ae5 --- /dev/null +++ b/.gitlab/openapi-generator.md @@ -0,0 +1,54 @@ +<!-- This file is being drafted here ahead of being merged into the guidebook. --> + +# OpenAPI client generation + +This document describes the use of the OpenAPI generator template. The OpenAPI generator template is +intended to be included in the "common" pipeline and so will not add any jobs to the pipeline unless +specifically triggered. + +## Basic use + +Add a file named `openapi.yaml` to the root of your repository with the OpenAPI schema in it. This +will trigger a run of the OpenAPI client generator. No additional CI configuration is needed. + +If you schema is stored in a different file, the `OPENAPI_GENERATOR_SCHEMA_PATH` CI variable may be +set to override the schema location. The variable should be set to a path relative to the repository +root which points to the schema file. + +## Generating the schema in CI + +If you generate your OpenAPI schema dynamically, you need to explicitly enable OpenAPI client +generation by setting the `OPENAPI_GENERATOR_ENABLED` variable. Additionally you'll need to override +the `openapi:schema` job to generate your schema and write it to the path stored in the +`OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH` variable. + +For example: + +```yaml +# .gitlab-ci.yml + +variables: + # We need to explicitly enable the OpenAPI generation since we don't have an openapi.yaml file in + # the root of our repository which would otherwise signal that. + OPENAPI_GENERATOR_ENABLED: "1" + +# We do not ship the OpenAPI schema in the repository and so the openapi:schema job needs to be +# overridden to use our schema generation command. +openapi:schema: + script: + - my-schema-generator --output=$OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH +``` + +Not that the existing `openapi:schema` job's `before_script` ensures that the directory containing +that path exists and so you do not need a `mkdir` or equivalent. + +## CI variables + +The following variables can be set to change the behaviour of the CI template. + +|Variable|Default|Description| +|-|-|-| +|`OPENAPI_GENERATOR_ENABLED`|*unset*|Set to non-empty value to *enable* OpenAPI client generation if not otherwise automatically triggered.| +|`OPENAPI_GENERATOR_DISABLED`|*unset*|Set to non-empty value to *disable* OpenAPI client generation if automatically triggered when it shouldn't be.| +|`OPENAPI_GENERATOR_SCHEMA_PATH`|`openapi.yaml`|Location of static OpenAPI schema file within the repository.| +|`OPENAPI_GENERATOR_ARTIFACT_DIR`|`openapi`|Directory in which generated artifacts are placed.| -- GitLab