.gitlab-ci.yml

@@ -14,3 +14,8 @@ variables:
   DAST_DISABLED: "1"
   BUILD_DISABLED: "1"
   CONTAINER_SCANNING_DISABLED: "1"
+
+python:tox:
+  variables:
+    # Before changing this line, make sure that the regex in renovate.json will still match.
+    TOX_ADDITIONAL_REQUIREMENTS: "poetry==1.8.3"

CHANGELOG.md

 # Changelog
 
+## [1.1.0](https://gitlab.developers.cam.ac.uk/uis/devops/tools/aws-helper/compare/1.0.3...1.1.0) (2024-09-25)
+
+
+### Features
+
+* add a basic test suite ([e566169](https://gitlab.developers.cam.ac.uk/uis/devops/tools/aws-helper/commit/e56616902c9a0f467c161f0ae3b9c88aa1e0f258))
+* add stub testing infrastructure ([12ef6e6](https://gitlab.developers.cam.ac.uk/uis/devops/tools/aws-helper/commit/12ef6e6f7e6cbca83132726c16951efd18441df9))
+* allow test suites to show full tracebacks ([944e146](https://gitlab.developers.cam.ac.uk/uis/devops/tools/aws-helper/commit/944e146fdf3d4c148a9a0cd815b55cd09002e78e))
+* mark code as compatible with earlier Pythons ([6d50214](https://gitlab.developers.cam.ac.uk/uis/devops/tools/aws-helper/commit/6d50214bb272dfc3e497cc871a7ca683285c8a9a))
+
+
+### Bug Fixes
+
+* add Python 3.9 compatibility ([eca5b53](https://gitlab.developers.cam.ac.uk/uis/devops/tools/aws-helper/commit/eca5b539d66499cc40d87a3099cbd1609fb68a59))
+
 ## [1.0.3](https://gitlab.developers.cam.ac.uk/uis/devops/tools/aws-helper/compare/1.0.2...1.0.3) (2024-09-25)
 
 

aws_helper/cli.py

@@ -4,7 +4,7 @@ Various helper functions for CLI commands.
 
 import sys
 from pathlib import Path
-from typing import Optional
+from typing import Optional, Union
 
 import yaml
 from rich.console import Console, Text
@@ -14,10 +14,18 @@ from .config import Config, ConfigNotFoundError, Environment, find_config
 
 # Consoles for reporting errors and informational messages. Both write to stderr since the only
 # thing which should be written to stdout is the console URL.
+exception_console = Console(markup=False, stderr=True, quiet=True)
 error_console = Console(markup=False, stderr=True, style="red")
 info_console = Console(markup=False, stderr=True)
 
 
+def set_show_exceptions(show_exceptions: bool):
+    """
+    Show exceptions for "expected" errors.
+    """
+    exception_console.quiet = not show_exceptions
+
+
 def set_quiet(quiet: bool):
     """
     Set quiet flag for info console.
@@ -25,7 +33,7 @@ def set_quiet(quiet: bool):
     info_console.quiet = quiet
 
 
-def load_config(config_file: Optional[str | Path]) -> Config:
+def load_config(config_file: Optional[Union[str, Path]]) -> Config:
     """
     Load application configuration.
 
@@ -38,6 +46,7 @@ def load_config(config_file: Optional[str | Path]) -> Config:
     try:
         config_path = find_config(config_file)
     except ConfigNotFoundError:
+        exception_console.print_exception(show_locals=True)
         error_console.print(
             "Failed to locate a configuration file. Either provide the location explicitly",
             "with the",
@@ -55,6 +64,7 @@ def load_config(config_file: Optional[str | Path]) -> Config:
         with config_path.open() as f:
             return Config.model_validate(yaml.safe_load(f))
     except Exception as e:
+        exception_console.print_exception(show_locals=True)
         error_console.print(
             "Error loading configuration from ",
             Text(f'"{config_path.as_posix()}"', "yellow"),
@@ -97,6 +107,7 @@ def google_id_token(environment_config: Environment) -> tuple[str, google.UserIn
         )
         user_info = google.get_user_info(google_creds)
     except Exception as e:
+        exception_console.print_exception(show_locals=True)
         error_console.print(f"Failed to get Google access token. Error was: {e}")
         error_console.print(
             "Hint: ensure that you have an up-to-date sign in token for an Google Cloud admin",
@@ -121,6 +132,7 @@ def google_id_token(environment_config: Environment) -> tuple[str, google.UserIn
             environment_config.aws_role_arn,
         )
     except Exception as e:
+        exception_console.print_exception(show_locals=True)
         error_console.print(
             "Failed to create id token for",
             Text(environment_config.google_service_account_email, "yellow"),
@@ -158,5 +170,6 @@ def aws_credentials(
             profile_name=profile_name,
         )
     except Exception as e:
+        exception_console.print_exception(show_locals=True)
         error_console.print(f"Failed to obtain AWS credentials. Error was: {e}")
         sys.exit(1)

aws_helper/config.py

 from pathlib import Path
-from typing import Annotated, Optional
+from typing import Annotated, Optional, Union
 
 from pydantic import BaseModel, Field
 
@@ -20,7 +20,7 @@ class ConfigNotFoundError(RuntimeError):
         super().__init__("No configuration file found")
 
 
-def find_config(config_file: Optional[str | Path] = None) -> Path:
+def find_config(config_file: Optional[Union[str, Path]] = None) -> Path:
     """
     Look for a configuration file starting from the current directory and walking upwards until we
     reach a file system boundary.

aws_helper/google.py

@@ -2,7 +2,8 @@ import urllib.parse
 from typing import Optional
 
 import requests
-from google.auth import default, impersonated_credentials
+from google import auth
+from google.auth import impersonated_credentials
 from google.auth.credentials import Credentials
 from google.auth.transport.requests import Request
 from pydantic import BaseModel
@@ -24,7 +25,7 @@ def get_default_credentials(quota_project_id: Optional[str] = None) -> Credentia
         Google credentials
 
     """
-    credentials, _ = default(quota_project_id=quota_project_id)
+    credentials, _ = auth.default(quota_project_id=quota_project_id)
     return credentials
 
 

pyproject.toml

 [tool.poetry]
 name = "aws-helper"
-version = "1.0.3"
+version = "1.1.0"
 description = "Utility to allow easy access to the AWS console and CLI tools"
 authors = [
   "University of Cambridge Information Services <devops@uis.cam.ac.uk>"
@@ -8,7 +8,7 @@ authors = [
 readme = "README.md"
 
   [tool.poetry.dependencies]
-  python = "^3.11"
+  python = "^3.9"
   requests = "^2.32.3"
   boto3 = "^1.35.10"
   google-auth = "^2.34.0"
@@ -20,6 +20,18 @@ readme = "README.md"
   [tool.poetry.scripts]
   aws-helper = "aws_helper:app"
 
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.3"
+pytest-cov = "^5.0.0"
+coverage = "^7.6.1"
+tox = "^4.20.0"
+faker = "^29.0.0"
+responses = "^0.25.3"
+
+  [tool.poetry.group.dev.dependencies.moto]
+  version = "^5.0.15"
+  extras = [ "sts" ]
+
 [tool.mypy]
 ignore_missing_imports = true
 
@@ -29,6 +41,9 @@ line-length = 99
 [tool.isort]
 profile = "black"
 
+[tool.coverage.run]
+source = [ "aws_helper" ]
+
 [build-system]
 requires = [ "poetry-core" ]
 build-backend = "poetry.core.masonry.api"

renovate.json

@@ -2,5 +2,17 @@
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
     "local>uis/devops/renovate-config"
+  ],
+  "customManagers": [
+    {
+      "customType": "regex",
+      "fileMatch": [
+        "^.gitlab-ci\\.yml$"
+      ],
+      "matchStrings": [
+        "TOX_ADDITIONAL_REQUIREMENTS:\\s*\"?(?<depName>.*?)==(?<currentValue>[0-9\\.]*)\"?"
+      ],
+      "datasourceTemplate": "pypi"
+    }
   ]
 }

tests/conftest.py

+import pytest
+import responses
+
+from .fixtures import *  # noqa: F401, F403
+
+
+@pytest.fixture(autouse=True)
+def requests_mock():
+    "Run all tests with requests being mocked."
+    with responses.RequestsMock() as mock:
+        yield mock

tests/fixtures/__init__.py

+from .aws import *  # noqa: F401, F403
+from .cli import *  # noqa: F401, F403
+from .google import *  # noqa: F401, F403

tests/fixtures/aws.py

+import pytest
+from moto import mock_aws
+
+
+@pytest.fixture(autouse=True)
+def fake_aws_credentials(monkeypatch):
+    """
+    Ensure no test will ever try to access real AWS resouces by providing invalid credentials.
+    """
+    monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing")
+    monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing")
+    monkeypatch.setenv("AWS_SECURITY_TOKEN", "testing")
+    monkeypatch.setenv("AWS_SESSION_TOKEN", "testing")
+    monkeypatch.setenv("AWS_DEFAULT_REGION", "eu-west-2")
+
+
+@pytest.fixture(autouse=True)
+def mocked_aws(fake_aws_credentials):
+    """Mock all AWS interactions using moto."""
+    with mock_aws():
+        yield

tests/fixtures/cli.py

+from collections.abc import Callable
+
+import pytest
+import yaml
+from typer.testing import CliRunner, Result
+
+from aws_helper import app, cli
+from aws_helper.config import Config, Environment
+
+
+@pytest.fixture(autouse=True)
+def show_exceptions():
+    "Configure aws_helper.cli to always show full exceptions to aid debugging."
+    cli.set_show_exceptions(True)
+
+
+@pytest.fixture
+def cli_runner() -> CliRunner:
+    return CliRunner(mix_stderr=False)
+
+
+@pytest.fixture
+def cli_invoke(cli_runner) -> Callable[..., Result]:
+    def f(*args):
+        return cli_runner.invoke(app, args)
+
+    return f
+
+
+@pytest.fixture
+def isolated_filesystem(cli_runner):
+    with cli_runner.isolated_filesystem():
+        yield
+
+
+@pytest.fixture
+def write_cli_config(isolated_filesystem) -> Callable[[Config], None]:
+    def f(config: Config):
+        with open(".aws-helper.yaml", "w") as fobj:
+            yaml.dump(config.model_dump(), fobj)
+
+    return f
+
+
+def _make_fake_environment(faker):
+    return Environment(
+        google_service_account_email=faker.email(),
+        google_quota_project_id=faker.slug(),
+        aws_role_arn=faker.bothify("arn:aws:iam::############:role/???????"),
+        aws_region=faker.slug(),
+    )
+
+
+@pytest.fixture
+def fake_cli_config(isolated_filesystem, write_cli_config, faker) -> Config:
+    config = Config(
+        environments={
+            "development": _make_fake_environment(faker),
+            faker.slug(): _make_fake_environment(faker),
+        }
+    )
+    write_cli_config(config)
+    return config

tests/fixtures/google.py

+import secrets
+from unittest import mock
+
+import pytest
+from google.auth.credentials import Credentials
+
+
+class MockCredentials(Credentials):
+    "Mock credentials object used in place of real Google credentials"
+
+    def __init__(self, token):
+        self._post_refresh_token = token
+
+    def refresh(self, request):
+        self.token = self._post_refresh_token
+
+
+@pytest.fixture
+def mock_google_default_token():
+    return secrets.token_urlsafe()
+
+
+@pytest.fixture
+def mock_google_impersonated_token():
+    return secrets.token_urlsafe()
+
+
+@pytest.fixture
+def mock_google_id_token():
+    return secrets.token_urlsafe()
+
+
+@pytest.fixture
+def mock_google_user_info(faker):
+    return {"id": faker.slug(), "email": faker.email()}
+
+
+@pytest.fixture
+def mock_google_user_info_endpoint(
+    mock_google_user_info, mock_google_default_token, requests_mock
+):
+    "Mock the https://www.googleapis.com/oauth2/v1/userinfo endpoint for the default user."
+    requests_mock.get(
+        f"https://www.googleapis.com/oauth2/v1/userinfo?access_token={mock_google_default_token}",
+        headers={"Content-Type": "application/json"},
+        json=mock_google_user_info,
+    )
+
+
+@pytest.fixture(autouse=True)
+def mock_google_auth_default(monkeypatch, faker, mock_google_default_token):
+    "Mock Google default credential provider."
+    magic_mock = mock.MagicMock()
+    magic_mock.return_value = (MockCredentials(mock_google_default_token), faker.slug())
+    monkeypatch.setattr("google.auth.default", magic_mock)
+    return magic_mock
+
+
+@pytest.fixture(autouse=True)
+def mock_google_auth_impersonated_credentials(monkeypatch, mock_google_impersonated_token):
+    "Mock Google impersonated credential provider."
+    magic_mock = mock.MagicMock()
+    magic_mock.return_value = MockCredentials(mock_google_impersonated_token)
+    monkeypatch.setattr("google.auth.impersonated_credentials.Credentials", magic_mock)
+    return magic_mock
+
+
+@pytest.fixture(autouse=True)
+def mock_google_auth_id_token_credentials(monkeypatch, mock_google_id_token):
+    "Mock Google impersonated credential provider."
+    magic_mock = mock.MagicMock()
+    magic_mock.return_value = MockCredentials(mock_google_id_token)
+    monkeypatch.setattr("google.auth.impersonated_credentials.IDTokenCredentials", magic_mock)
+    return magic_mock

tests/test_console.py

+import json
+import re
+import secrets
+import urllib.parse
+
+import pytest
+import responses
+
+
+@pytest.fixture
+def mock_get_signin_token(requests_mock):
+    def request_callback(request):
+        headers = {"Content-Type": "application/json"}
+        token = secrets.token_urlsafe()
+        return (200, headers, json.dumps({"SigninToken": token}))
+
+    requests_mock.add_callback(
+        responses.GET,
+        re.compile("https://[a-z0-9-]+\\.signin\\.aws\\.amazon\\.com/federation"),
+        callback=request_callback,
+        content_type="application/json",
+        match=[
+            responses.matchers.query_param_matcher(
+                {"Action": "getSigninToken"}, strict_match=False
+            )
+        ],
+    )
+
+
+def test_configuration_file_required(cli_invoke):
+    "Command fails if no configuration file is present."
+    result = cli_invoke("console")
+    assert result.exit_code != 0
+    assert "Failed to locate a configuration file" in result.stderr
+
+
+def test_basic_usage(
+    cli_invoke, fake_cli_config, mock_google_user_info_endpoint, mock_get_signin_token
+):
+    result = cli_invoke("console", "--no-browser")
+    assert result.exit_code == 0
+    parts = urllib.parse.urlsplit(result.stdout.strip())
+    assert parts.scheme == "https"
+    assert parts.netloc.endswith(".signin.aws.amazon.com")
+    assert parts.path == "/federation"
+    query = urllib.parse.parse_qs(parts.query)
+    assert query["Action"] == ["login"]

tests/test_credential_helper.py

+import json
+
+
+def test_configuration_file_required(cli_invoke):
+    "Command fails if no configuration file is present."
+    result = cli_invoke("cli-credentials")
+    assert result.exit_code != 0
+    assert "Failed to locate a configuration file" in result.stderr
+
+
+def test_basic_usage(cli_invoke, fake_cli_config, mock_google_user_info_endpoint):
+    result = cli_invoke("cli-credentials")
+    assert result.exit_code == 0
+    creds = json.loads(result.stdout)
+    assert creds["Version"] == 1
+    assert "AccessKeyId" in creds
+    assert "SecretAccessKey" in creds
+    assert "Expiration" in creds

tests/test_import.py

+def test_import_succeeds():
+    "Importing the aws_helper module should not have side-effects or errors."
+    import aws_helper  # noqa: F401

tox.ini

+# Tox runner configuration
+#
+# The following optional environment variables can change behaviour. See the
+# comments where they are used for more information.
+#
+# - TOXINI_ARTEFACT_DIR
+#
+[tox]
+envlist=py3
+isolated_build=True
+
+# The "_vars" section is ignored by tox but we place some useful shared
+# variables in it to avoid needless repetition.
+[_vars]
+build_root={env:TOXINI_ARTEFACT_DIR:{toxinidir}/build}
+
+[testenv:py3]
+basepython=python3
+skip_install=True
+allowlist_externals=
+    poetry
+commands_pre=
+    poetry install --sync
+commands=
+    pytest --cov --cov-config=pyproject.toml --junitxml={[_vars]build_root}/{envname}/junit.xml {posargs}
+    coverage html --directory {[_vars]build_root}/{envname}/htmlcov/
+    coverage xml -o {[_vars]build_root}/{envname}/coverage.xml