diff --git a/CHANGELOG.md b/CHANGELOG.md index 411307abe06224dc43c51f26c2a148f77dd66b7c..13efe5d2b6eee410552c94f3f967d672775bd656 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [6.3.0] - 2024-12-05 + +### Added + +- Added a new feature to auto-generate API clients from OpenAPI specifications. +- Enabled the OpenAPI client generation feature as part of the common pipeline. + ## [6.2.0] - 2024-11-21 ### Added diff --git a/auto-devops/common-pipeline.yml b/auto-devops/common-pipeline.yml index 2d8cc0dbba136d87e18edcd82fe7fa45ae0c5255..f3b77ec8a983f86d996096b7524af2ef676f8c72 100644 --- a/auto-devops/common-pipeline.yml +++ b/auto-devops/common-pipeline.yml @@ -19,6 +19,7 @@ include: - local: "/auto-devops/python-check-tags-match-version.yml" - local: "/auto-devops/mkdocs-docs.gitlab-ci.yml" - local: "/auto-devops/trigger-renovatebot.gitlab-ci.yml" + - local: "/auto-devops/openapi-generator.gitlab-ci.yml" # Fail-safe workflow rules. These can be overridden by CI configuration which includes us. - template: Workflows/Branch-Pipelines.gitlab-ci.yml diff --git a/auto-devops/openapi-generator.gitlab-ci.yml b/auto-devops/openapi-generator.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..229c95a95e1dec26cb6a029bf4f88ce39057175b --- /dev/null +++ b/auto-devops/openapi-generator.gitlab-ci.yml @@ -0,0 +1,390 @@ +# Reusable template for generating API clients from an OpenAPI schema. +# +# See the openapi-generator.md file in this directory for detailed usage information. +# +# This template is intended to be included in the common pipeline and so follows the common pipeline +# rules: +# +# - No jobs are added unless triggered explicitly or via the presence of a special file. +# - All jobs can be disabled if incorrectly triggered. +# - All "public" CI variables are namespaced to avoid collisions. +# - All build artefacts are added to a configurable directory to allow customisation if that +# directory conflicts with one in the repository. +# +# This template requires the Auto DevOps stages. + +variables: + # Location of the OpenAPI schema file within the repository. + OPENAPI_GENERATOR_SCHEMA_PATH: "openapi.yaml" + + # Location of various job artefacts. These will always be relative to $CI_PROJECT_DIR. + OPENAPI_GENERATOR_ARTIFACT_DIR: "openapi" + OPENAPI_GENERATOR_SOURCE_ARTIFACT_DIR: $OPENAPI_GENERATOR_ARTIFACT_DIR/src + OPENAPI_GENERATOR_PACKAGE_ARTIFACT_DIR: $OPENAPI_GENERATOR_ARTIFACT_DIR/packages + OPENAPI_GENERATOR_DOCS_ARTIFACT_DIR: $OPENAPI_GENERATOR_ARTIFACT_DIR/docs + OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH: $OPENAPI_GENERATOR_ARTIFACT_DIR/schema.yml + OPENAPI_GENERATOR_SCHEMA_VERSION_ARTIFACT: $OPENAPI_GENERATOR_ARTIFACT_DIR/schema-version + + # Generator-specific package names. These default to the project name but can be customised on a + # per-generator basis. + OPENAPI_GENERATOR_PACKAGE_NAME: $CI_PROJECT_NAME + OPENAPI_GENERATOR_TYPESCRIPT_AXIOS_PACKAGE_NAME: $OPENAPI_GENERATOR_PACKAGE_NAME + OPENAPI_GENERATOR_PYTHON_URLLIB3_PACKAGE_NAME: $OPENAPI_GENERATOR_PACKAGE_NAME + + # Suffix added to package version number. Can be overridden globally or per-job. We default to + # adding semver-style build information. + OPENAPI_GENERATOR_PACKAGE_VERSION_SUFFIX: "+$CI_JOB_ID.$CI_COMMIT_SHORT_SHA" + + # Specify Docker images used by jobs. + OPENAPI_GENERATOR_YQ_IMAGE: mikefarah/yq:4 + OPENAPI_GENERATOR_OPENAPI_GENERATOR_IMAGE: openapitools/openapi-generator-cli:v7.10.0 + OPENAPI_GENERATOR_NODE_IMAGE: node:lts-slim + OPENAPI_GENERATOR_PYTHON_IMAGE: $PYTHON_IMAGE + + # Extra arguments to openapi-generator-cli generate command. Can be overridden per-job if + # necessary. + OPENAPI_GENERATOR_GENERATOR_EXTRA_ARGS: "" + + # Ref name where packages should be published from. Can be overridden by specialising the rules on + # ".openapi:publish:base". + OPENAPI_GENERATOR_PUBLISH_REF_NAME: $CI_DEFAULT_BRANCH + +# Ensure that the schema is present as the $OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH artefact. The +# default behaviour is to copy the file specified by $OPENAPI_GENERATOR_SCHEMA_PATH but the job can +# be extended for more complex behaviour or dynamically generated schema. +openapi:schema: + stage: build + needs: [] + before_script: + # Ensure that the directory which is to contain the schema exists. + - mkdir -p $(dirname "$OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH") + script: + # The default behaviour is to copy the schema file to the artifact directory. + - cp "$OPENAPI_GENERATOR_SCHEMA_PATH" "$OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH" + artifacts: + paths: + - $OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH + # When extending this job, rules which *disable* jobs should be included *above* these rules. + # Rules which *enable* the job should be included *below* them. + rules: + # Never run any OpenAPI generate jobs if OPENAPI_GENERATOR_DISABLED is set. + - if: $OPENAPI_GENERATOR_DISABLED + when: never + # If no API specification is provided, never run. + - if: ($OPENAPI_GENERATOR_SCHEMA_PATH == null) || ($OPENAPI_GENERATOR_SCHEMA_PATH == "") + when: never + # If an OpenAPI spec is present in the repository and it has changed, use it. + - if: $OPENAPI_GENERATOR_SCHEMA_PATH + changes: + - $OPENAPI_GENERATOR_SCHEMA_PATH + exists: + paths: + - $OPENAPI_GENERATOR_SCHEMA_PATH + # Finally, allow OpenAPI generation to be explicitly enabled if desired. + - if: $OPENAPI_GENERATOR_ENABLED + +# Extract the version from the OpenAPI schema. This is required by some generator jobs as some +# generator templates do not respect the version set in the schema. The version is written to +# $OPENAPI_GENERATOR_SCHEMA_VERSION_ARTIFACT. Note that even generators which support reading the +# version from the schema should respect the artefact generated by this job since it provides a +# customisation point to override the package version if needed. +openapi:schema:version: + stage: build + needs: ["openapi:schema"] + image: + name: $OPENAPI_GENERATOR_YQ_IMAGE + entrypoint: [""] + before_script: + - mkdir -p $(dirname "$OPENAPI_GENERATOR_SCHEMA_VERSION_ARTIFACT") + script: + - yq -r ".info.version" "$OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH" >"$OPENAPI_GENERATOR_SCHEMA_VERSION_ARTIFACT" + artifacts: + paths: + - $OPENAPI_GENERATOR_SCHEMA_VERSION_ARTIFACT + rules: + - !reference ["openapi:schema", rules] + +# Template CI job for running openapi-generator. +.openapi:generator-cli: + image: + name: $OPENAPI_GENERATOR_OPENAPI_GENERATOR_IMAGE + entrypoint: [""] + variables: + # Customise how openapi-generator-cli is run. + OPENAPI_GENERATOR_CMD: "/usr/local/bin/docker-entrypoint.sh" + rules: + - !reference ["openapi:schema", rules] + +# Validate the OpenAPI schema. +openapi:schema:validate: + extends: ".openapi:generator-cli" + stage: test + needs: ["openapi:schema"] + script: + - |- + $OPENAPI_GENERATOR_CMD validate \ + --input-spec "$OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH" \ + --recommend + +### GENERATOR-SPECIFIC BASE TEMPLATES ### + +.openapi:generator:base: + variables: + _generator_name: "" # required + _generator_slug: "" # required + _package_name: $OPENAPI_GENERATOR_PACKAGE_NAME # optional + _package_version_additional_property: "" # optional + rules: + - !reference ["openapi:schema", rules] + +.openapi:generator:typescript-axios: + extends: [".openapi:generator:base"] + variables: + _package_name: $OPENAPI_GENERATOR_TYPESCRIPT_AXIOS_PACKAGE_NAME + _generator_slug: typescript-axios + _generator_name: typescript-axios + rules: + # Allow explicitly disabling the client if it would otherwise be enabled. + - if: $OPENAPI_GENERATOR_TYPESCRIPT_AXIOS_DISABLED + when: never + - !reference [".openapi:generator:base", rules] + +.openapi:generator:python-urllib3: + extends: [".openapi:generator:base"] + variables: + _package_name: $OPENAPI_GENERATOR_PYTHON_URLLIB3_PACKAGE_NAME + _generator_slug: python-urllib3 + _generator_name: python + rules: + # Allow explicitly disabling the client if it would otherwise be enabled. + - if: $OPENAPI_GENERATOR_PYTHON_URLLIB3_DISABLED + when: never + - !reference [".openapi:generator:base", rules] + +#### GENERATION OF PACKAGE SOURCE CODE #### + +# Template CI job to generate an API client or server. +.openapi:generate: + extends: [".openapi:generator:base", ".openapi:generator-cli"] + stage: build + needs: ["openapi:schema", "openapi:schema:version"] + variables: + _generator_args: "" # optional + before_script: + - mkdir -p $(dirname "$OPENAPI_GENERATOR_SOURCE_ARTIFACT_DIR") + script: + - |- + if [ ! -z "$_package_version_additional_property" ]; then + # This is a bit of a hack in order to interpolate packageVersion *after* the schema version + # artifact is present. + _package_version_args="--additional-properties ${_package_version_additional_property}=$(cat $OPENAPI_GENERATOR_SCHEMA_VERSION_ARTIFACT)$OPENAPI_GENERATOR_PACKAGE_VERSION_SUFFIX" + fi + $OPENAPI_GENERATOR_CMD generate \ + --input-spec "$OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH" \ + --output "$OPENAPI_GENERATOR_SOURCE_ARTIFACT_DIR/$_generator_slug/$_package_name" \ + --generator-name "$_generator_name" \ + $_package_version_args \ + $_generator_args \ + $OPENAPI_GENERATOR_GENERATOR_EXTRA_ARGS + artifacts: + paths: + - $OPENAPI_GENERATOR_SOURCE_ARTIFACT_DIR/$_generator_slug/$_package_name + rules: + - !reference ["openapi:schema", rules] + +# Generate a TypeScript + Axios based client. +openapi:generate:typescript-axios: + extends: [".openapi:generate", ".openapi:generator:typescript-axios"] + variables: + _generator_args: "--additional-properties npmName=$OPENAPI_GENERATOR_TYPESCRIPT_AXIOS_PACKAGE_NAME" + _package_version_additional_property: "npmVersion" + +# Generate a Python + urllib3 client. +openapi:generate:python-urllib3: + extends: [".openapi:generate", ".openapi:generator:python-urllib3"] + variables: + _generator_name: python + _generator_args: >- + --additional-properties library=urllib3 + --additional-properties packageName=$OPENAPI_GENERATOR_PYTHON_URLLIB3_PACKAGE_NAME + _package_version_additional_property: "packageVersion" + +#### PACKAGING SOURCE CODE INTO PACKAGE ARTEFACTS #### + +# Prepare to build packaging artefacts by ensuring the destination directory exists and cd-ing to +# the package's source directory. +.openapi:build:base: + extends: ".openapi:generator:base" + stage: build + before_script: + - mkdir -p "$OPENAPI_GENERATOR_PACKAGE_ARTIFACT_DIR/$_generator_slug" + - cd "$OPENAPI_GENERATOR_SOURCE_ARTIFACT_DIR/$_generator_slug/$_package_name" + artifacts: + paths: + - $OPENAPI_GENERATOR_PACKAGE_ARTIFACT_DIR/$_generator_slug + +# Template CI job to build a npm package into a publishable tarball. +.openapi:build:npm: + extends: ".openapi:build:base" + image: $OPENAPI_GENERATOR_NODE_IMAGE + before_script: + - !reference [".openapi:build:base", before_script] + - npm install + script: + - export npm_config_pack_destination="$CI_PROJECT_DIR/$OPENAPI_GENERATOR_PACKAGE_ARTIFACT_DIR/$_generator_slug/" + - npm pack + +# Template CI job to build a Python source tarball and wheel. +.openapi:build:python: + extends: ".openapi:build:base" + image: $OPENAPI_GENERATOR_PYTHON_IMAGE + before_script: + - !reference [".openapi:build:base", before_script] + - pip install build + script: + - python -m build + - mv dist/* "$CI_PROJECT_DIR/$OPENAPI_GENERATOR_PACKAGE_ARTIFACT_DIR/$_generator_slug/" + +# Build a TypeScript + Axios client. +openapi:build:typescript-axios: + extends: [".openapi:build:npm", ".openapi:generator:typescript-axios"] + needs: ["openapi:generate:typescript-axios"] + +# Build a Python + urllib3 client. +openapi:build:python-urllib3: + extends: [".openapi:build:python", ".openapi:generator:python-urllib3"] + needs: ["openapi:generate:python-urllib3"] + +#### GENERATING DOCUMENTATION #### + +.openapi:doc:base: + extends: ".openapi:build:base" + before_script: + - mkdir -p $(dirname "$OPENAPI_GENERATOR_DOCS_ARTIFACT_DIR") + - !reference [".openapi:build:base", before_script] + artifacts: + paths: + - $OPENAPI_GENERATOR_DOCS_ARTIFACT_DIR/$_generator_slug + +# Template CI job to generate typedoc documentation from an JavaScript/TypeScript package. +.openapi:docs:typedoc: + extends: ".openapi:doc:base" + image: $OPENAPI_GENERATOR_NODE_IMAGE + variables: + _typedoc_entry_points: "" # required + _typedoc_extra_args: "" # optional + before_script: + - !reference [".openapi:doc:base", before_script] + - npm install + script: + - |- + npx typedoc \ + --out "$CI_PROJECT_DIR/$OPENAPI_GENERATOR_DOCS_ARTIFACT_DIR/$_generator_slug" \ + $_typedoc_extra_args $_typedoc_entry_points + +# Template CI job to generate pdoc documentation from a Python package. +.openapi:docs:pdoc: + extends: ".openapi:doc:base" + image: $OPENAPI_GENERATOR_PYTHON_IMAGE + variables: + _pdoc_module: $_package_name # optional + _pdoc_extra_args: "" # optional + before_script: + - !reference [".openapi:doc:base", before_script] + - pip install pdoc + - pip install -e . + script: + - |- + pdoc \ + --output-directory "$CI_PROJECT_DIR/$OPENAPI_GENERATOR_DOCS_ARTIFACT_DIR/$_generator_slug" \ + $_pdoc_extra_args $_pdoc_module + +# Generate documentation from the OpenAPI schema using redoc. +openapi:docs:redoc: + stage: build + needs: ["openapi:schema"] + image: $OPENAPI_GENERATOR_NODE_IMAGE + script: + - mkdir -p "$CI_PROJECT_DIR/$OPENAPI_GENERATOR_DOCS_ARTIFACT_DIR/redoc" + - |- + npx @redocly/cli build-docs \ + "--output=$CI_PROJECT_DIR/$OPENAPI_GENERATOR_DOCS_ARTIFACT_DIR/redoc/index.html" \ + "$OPENAPI_GENERATOR_SCHEMA_ARTIFACT_PATH" + artifacts: + expose_as: "API documentation" + paths: + - $OPENAPI_GENERATOR_DOCS_ARTIFACT_DIR/redoc + rules: + # Allow explicitly disabling redoc documentation if desired. + - if: $OPENAPI_GENERATOR_REDOC_DISABLED + when: never + - !reference ["openapi:schema", rules] + +# Generate documentation for a TypeScript + Axios client. +openapi:docs:typescript-axios: + extends: [".openapi:docs:typedoc", ".openapi:generator:typescript-axios"] + needs: ["openapi:generate:typescript-axios"] + variables: + _typedoc_entry_points: "index.ts" + +# Generate documentation for a Python + urllib3 client. +openapi:docs:python-urllib3: + extends: [".openapi:docs:pdoc", ".openapi:generator:python-urllib3"] + needs: ["openapi:generate:python-urllib3"] + +#### PUBLISHING GENERATED PACKAGES #### + +# Base CI template for publish jobs. +.openapi:publish:base: + extends: ".openapi:generator:base" + stage: production + before_script: + - cd "$OPENAPI_GENERATOR_PACKAGE_ARTIFACT_DIR/$_generator_slug" + rules: + - if: $OPENAPI_GENERATOR_PUBLISH_DISABLED + when: never + - if: $CI_COMMIT_REF_NAME != $OPENAPI_GENERATOR_PUBLISH_REF_NAME + when: never + - !reference [".openapi:generator:base", rules] + +# Base CI template for publishing npm packages to GitLab's own package registry. +.openapi:publish:gitlab:npm: + extends: ".openapi:publish:base" + image: $OPENAPI_GENERATOR_NODE_IMAGE + script: + - echo "registry=https://${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/" >> .npmrc + - echo "//${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN}" >> .npmrc + - npm publish *.tgz + rules: + - if: $OPENAPI_GENERATOR_PUBLISH_GITLAB_DISABLED + when: never + - !reference [".openapi:publish:base", rules] + +# Base CI template for publishing Python packages to GitLab's own package registry. +.openapi:publish:gitlab:python: + extends: ".openapi:publish:base" + image: $OPENAPI_GENERATOR_PYTHON_IMAGE + variables: + TWINE_USERNAME: gitlab-ci-token + TWINE_PASSWORD: $CI_JOB_TOKEN + script: + - pip install twine + - python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi * + rules: + - if: $OPENAPI_GENERATOR_PUBLISH_GITLAB_DISABLED + when: never + - !reference [".openapi:publish:base", rules] + +# Publish a TypeScript + Axios client. +openapi:publish:typescript-axios:gitlab: + extends: [".openapi:publish:gitlab:npm", ".openapi:generator:typescript-axios"] + needs: ["openapi:build:typescript-axios"] + rules: + - !reference [".openapi:publish:gitlab:npm", rules] + +# Publish a Python + urllib3 client +openapi:publish:python-urllib3: + extends: [".openapi:publish:gitlab:python", ".openapi:generator:python-urllib3"] + needs: ["openapi:build:python-urllib3"] + rules: + - !reference [".openapi:publish:gitlab:python", rules]