Merge branch 'initial-framework' into 'master'

Sync Tool Framework

Closes #2

See merge request !1

.coveragerc

+[run]
+omit =
+	.tox/*
+	setup.py
+  .venv/*

.dockerignore

+.git
+venv/
+
+*.pyc
+__pycache__/
+
+.pytest_cache/
+.coverage
+htmlcov/
+
+dist/
+build/
+*.egg-info/

.flake8

+[flake8]
+max-line-length=99
+exclude=.venv,venv,.tox

.gitignore

+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+.static_storage/
+.media/
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+*.sqlite3
+
+# PyCharm
+.idea
+
+# Service account credentials (if instructions in README are followed)
+credentials.json
+
+# Local configuration
+*.yaml
+*.yml
+!.gitlab-ci.yml
+!configuration-example.yaml
+!secrets-example.yaml

.gitlab-ci.yml

+# This file pulls in the GitLab AutoDevOps configuration via an include
+# directive and then overrides bits. The rationale for this is we'd like this
+# file to eventually have zero local overrides so that we can use the AutoDevOps
+# pipeline as-is.
+
+include:
+  # Bring in the AutoDevOps template from GitLab.
+  # It can be viewed at:
+  # https://gitlab.com/gitlab-org/gitlab-ee/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+  - template: Auto-DevOps.gitlab-ci.yml
+
+  # Overrides to AutoDevOps for testing
+  - project: 'uis/devops/continuous-delivery/ci-templates'
+    file: '/auto-devops/tox-tests.yml'
+
+variables:
+  DOCUMENTATION_DISABLED: "1"

Dockerfile

+FROM uisautomation/python:3.7-alpine
+
+WORKDIR /usr/src/app
+
+# Install specific requirements for the package.
+ADD requirements.txt ./
+RUN pip install tox && pip install -r requirements.txt
+
+# Copy application source and install it. Use "-e" to avoid needlessly copying
+# files into the site-packages directory.
+ADD ./ ./
+RUN pip install -e .
+
+ENTRYPOINT ["essmsync"]

LICENSE

+MIT License
+
+Copyright (c) 2020 University of Cambridge Information Services
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

 # Sync Tool
 
-Tool for performing all ESSM sync operations
\ No newline at end of file
+Tool for performing all Education Space Scheduling and Modelling (ESSM)
+synchronisation operations.
+
+Uses TermTime and Booker APIs to sync rooms, buildings, bookings, etc between
+these systems.
+
+Configuration is provided by one or more configuration files. This allows
+separate of services (say Booker production and staging configuration) and
+allows secrets to be held separately from non-secret configuration. These config
+files are deep merged together.
+
+Example [non-secret](configuration-example.yaml) and [secret](secret-example.yaml)
+files show required entries.
+
+## Installation
+
+The command-line tool can be installed directly from the git repository:
+```bash
+$ pip3 install git+https://gitlab.developers.cam.ac.uk/uis/devops/essm/sync-tool.git
+```
+For developers, the script can be installed from a cloned repo using pip:
+```bash
+$ cd /path/to/this/repo
+$ pip3 install -e .
+```
+
+## Usage
+
+The tool can be invoked from the command line:
+```bash
+$ essmsync
+```
+By default this will log what will be done. To actually perform the sync:
+```bash
+$ essmsync --really-do-this
+```
+To specify configuration files to be loaded the use the `--configuration` or `-c`
+arguments either multiple times or with a comma separated list of files (or a combination of both).
+```bash
+# multiple `-c` arguments
+$ essmsync -c termtime-test.yaml -c booker-staging.yaml
+# or comma separated
+$ essmsync -c termtime-test.yaml,booker-staging.yaml
+```
+>If not configuration files are specified then the tool will attempt to load a
+`configuration.yaml` and `secrets.yaml`.
+
+A list of operations (see [below](#operations)) can be specified at the end of
+the command line.
+
+See the output of `essmsync --help` for more information.
+
+### Docker Usage
+
+```bash
+$ docker run --rm -ti -v $(pwd):/usr/src/app \
+    registry.gitlab.developers.cam.ac.uk/uis/devops/essm/sync-tool/master \
+    -c termtime-test.yaml -c booker-staging.yaml \
+    test(inside,docker)
+```
+
+
+## Operations
+
+The operations for the sync tool to perform can be specified either in the
+`operations` list in the configuration root or appended to the command line.
+If both, then the command line options are appended to the configuration list.
+```bash
+$ cat configuration.yaml
+...
+operations:
+  - test(do,first)
+...
+$ essmsync --quiet -c configuration.yaml "test(do,next)" test "test(and,last)"
+Test called with ('do', 'first')
+Test called with ('do', 'next')
+Test called without arguments
+Test called with ('and', 'last')
+```
+> Note that operations with arguments are likely to need quoting to avoid
+conflicts with the shell parser
+
+### Dumping results
+
+Results from API queries (or any part of the configuration) can be dumped out
+in either yaml or json using the `dump_yaml` (shorthand `dump`) or `dump_json`
+operations. Both these take the 'key' in the configuration to dump and an optional
+flag as whether to include the parents of the key in the output. `dump_json` also
+optionally can be instructed to indent the output.
+
+```bash
+# Examples use the termtime get_rooms operation then dump the result
+
+# dump yaml including parent keys (the default)
+$ essmsync --quiet "termtime:get_rooms" "dump(termtime.rooms)"
+$ essmsync --quiet "termtime:get_rooms" "dump_yaml(termtime.rooms)"
+
+# dump yaml not including parent keys
+$ essmsync --quiet "termtime:get_rooms" "dump(termtime.rooms,false)"
+$ essmsync --quiet "termtime:get_rooms" "dump_yaml(termtime.rooms,false)"
+
+# dump json including parent keys
+$ essmsync --quiet "termtime:get_rooms" "dump_json(termtime.rooms,true)"
+
+# dump json not including parent keys (the default)
+$ essmsync --quiet "termtime:get_rooms" "dump_json(termtime.rooms)"
+
+# dump json not including parent keys with indent of 4 spaces
+$ essmsync --quiet "termtime:get_rooms" "dump_json(termtime.rooms,false,4)"
+```
+> The first example, produces yaml that could be included as a configuration file
+in another call to `essmsync`.

configuration-example.yaml

+# Example ESSM sync configuration
+#
+# Multiple configuration files can be specified and will be 'deep-merged'
+termtime:
+  api:
+    url: "https://domain/api/path/"
+
+booker:
+  api:
+    url: "https://domain/"
+
+operations:
+  - test(do,first)

essmsync/__init__.py

+"""
+Education Space Scheduling and Modelling synchronisation tool
+
+Usage:
+    essmsync (-h | --help)
+    essmsync [--configuration=FILE]... [--quiet] [--debug] [--really-do-this] [<operation>]...
+
+Options:
+    -h, --help                  Show a brief usage summary.
+
+    --quiet                     Reduce logging verbosity.
+    --debug                     Log debugging information
+
+    -c, --configuration=FILE    Specify configuration file to load. Multiple
+                                files will be merged (latter taking precedence).
+
+    --really-do-this            Actually try to make the changes.
+
+"""
+import logging
+import os
+import sys
+import http
+
+import docopt
+
+from . import config
+from . import sync
+
+
+LOG = logging.getLogger(os.path.basename(sys.argv[0]))
+
+
+def main():
+    opts = docopt.docopt(__doc__)
+
+    # Configure logging
+    logging.basicConfig(
+        level=logging.DEBUG if opts['--debug'] else
+        logging.WARN if opts['--quiet'] else logging.INFO
+    )
+
+    # Debug logging (don't use in production - secrets are displayed)
+    if opts['--debug']:
+        # Also enable requests debug logging
+        http.client.HTTPConnection.debuglevel = 1
+        req_log = logging.getLogger('requests.packages.urllib3')
+        req_log.setLevel(logging.DEBUG)
+        req_log.propagate = True
+
+    config_files = expand_lists(opts['--configuration'])
+    if len(config_files) == 0:
+        # default to attempting to load configuration.yaml and secrets.yaml
+        config_files = ['configuration.yaml', 'secrets.yaml']
+    config.load_configs(config_files)
+
+    # Add read only flag to configuration
+    config.set('read_only', not opts['--really-do-this'])
+
+    # Add any operations specified on command line
+    config.set('operations', opts['<operation>'])
+
+    # Perform sync actions
+    sync.sync()
+
+
+def expand_lists(list):
+    """
+    Iterate over list, if any comma-separated entries are found then split them
+    inserting them in place.
+    >>> expand_lists(['a', 'b'])
+    ['a', 'b']
+    >>> expand_lists(['a,b'])
+    ['a', 'b']
+    >>> expand_lists(['a,b', 'c', 'd'])
+    ['a', 'b', 'c', 'd']
+    >>> expand_lists(['a', 'b,c', 'd'])
+    ['a', 'b', 'c', 'd']
+    >>> expand_lists(['a', 'b', 'c,d'])
+    ['a', 'b', 'c', 'd']
+    >>> expand_lists(['a', 'b,c,d', 'e'])
+    ['a', 'b', 'c', 'd', 'e']
+    """
+    return [y for x in list for y in x.split(',')]

essmsync/__main__.py

+from . import main
+
+if __name__ == '__main__':
+    main()

essmsync/booker.py

+"""
+Handle authentication and usage of Booker API
+
+"""
+import logging
+
+from . import config
+from . import utils
+from .exceptions import BookerAPIError
+
+
+LOG = logging.getLogger(__name__)
+
+
+def booker_get(resource, **kwargs):
+    return utils.api_get(
+        config.get('booker.api.url') + resource,
+        exception=BookerAPIError,
+        headers=booker_headers(),
+        **kwargs
+    )
+
+
+def booker_post(resource, form=False, **kwargs):
+    return utils.api_post(
+        config.get('booker.api.url') + resource,
+        exception=BookerAPIError,
+        headers=booker_headers(form),
+        **kwargs
+    )
+
+
+def booker_headers(form=False):
+    h = {'user-agent': config.USER_AGENT}
+    if not form:
+        h['Authorization'] = 'Bearer ' + config.get('booker.api.access_token')
+    return h
+
+
+def auth():
+    LOG.info('Authenticating Booker API')
+
+    # Only request an access token if we don't already have one
+    if not config.get('booker.api.access_token'):
+        (token_resp, _) = booker_post(
+            'Token',
+            # Token endpoint only takes form-data not json!
+            form=True,
+            data={
+                'grant_type': 'password',
+                'username': config.get('booker.api.username'),
+                'password': config.get('booker.api.password')
+            }
+        )
+        access_token = token_resp.get('access_token')
+        if not access_token:
+            LOG.error('Booker API authentication failed')
+            raise BookerAPIError()
+        LOG.info('Booker API access token retrieved')
+        config.set('booker.api.access_token', access_token)
+    else:
+        LOG.info('Booker API access token already provided')
+
+    # No ping to validate access_token (so use roomtypes as a low impact)
+    (roomtypes, _) = booker_get('api/roomtypes')
+    # Should have a non-empty list back
+    if not isinstance(roomtypes, list) or len(roomtypes) == 0:
+        LOG.error('Booker API authentication failed')
+        raise BookerAPIError()
+    LOG.info('Booker API authentication succeeded')
+
+
+# Booker operations
+
+def op_get_rooms():
+    """
+    Get list of all rooms and put in `termtime.rooms`
+
+    """
+    (rooms, _) = booker_get('api/rooms')
+    if not isinstance(rooms, list):
+        LOG.error('Booker rooms list is not a list')
+        raise BookerAPIError()
+    # TODO: needs filtering to only rooms with 'TermTime' equipment note
+    config.set('booker.rooms', rooms, True)

essmsync/config.py

+"""
+Utilities for handling configuration files
+
+"""
+import logging
+import os
+from functools import reduce
+
+import yaml
+from deepmerge import always_merger
+
+from .exceptions import ConfigurationNotFound, ConfigurationInvalid
+
+
+LOG = logging.getLogger(__name__)
+
+# User agent for API requests
+USER_AGENT = 'essmsync'
+
+# Keys required in merged configuration
+REQUIRED_CONFIG_KEYS = [
+    'termtime.api.url',
+    'termtime.api.key',
+    'booker.api.url',
+    'booker.api.username',
+    'booker.api.password',
+]
+
+# Start with an empty configuration
+configuration = dict()
+
+
+def load_configs(config_files):
+    """
+    Load a list of configuration files and check required keys exist
+    """
+    LOG.info('Loading configurations')
+
+    # Check for missing configuration files
+    missing_files = [f for f in config_files if not os.path.isfile(f)]
+    if len(missing_files) > 0:
+        for f in missing_files:
+            LOG.error(f'"{f}" configuration missing')
+        raise ConfigurationNotFound()
+
+    # Load each configuration file
+    for f in config_files:
+        load_config(f)
+
+    # Check for missing required keys
+    invalid = False
+    for r in REQUIRED_CONFIG_KEYS:
+        value = get(r, None)
+        if value is None or value.strip() == "":
+            LOG.error(f'{r}: required but not set')
+            invalid = True
+    if invalid:
+        raise ConfigurationInvalid()
+
+    # All configuration loaded
+    return configuration
+
+
+def load_config(file):
+    """
+    Load an individual configuration file, deep merging it with existing configuration
+    """
+    LOG.info(f'Loading "{file}"')
+    with open(file) as f:
+        c = yaml.safe_load(f)
+        always_merger.merge(configuration, c)
+
+
+def get(dottedkey, default=None):
+    """
+    Get the value from a nested dict using a dotted key reference (e.g. key1.key2.key3)
+
+    >>> set(None, {'a':{'b':'c','d':{'e':'f'}},'g':'h'})
+    >>> get('g')
+    'h'
+    >>> get('a.b')
+    'c'
+    >>> get('a.d.e')
+    'f'
+    >>> get('x', 'missing')
+    'missing'
+    >>> get('a.x', 'missing')
+    'missing'
+    >>> get('a.d.x', 'missing')
+    'missing'
+    >>> get('a.x.y', 'missing')
+    'missing'
+    """
+    return reduce(
+        lambda d, k: d.get(k, default) if isinstance(d, dict) else default,
+        dottedkey.split('.'),
+        configuration
+    )
+
+
+def set(dottedkey, value=None, overwrite=False):
+    """
+    Set the value in a nested dict using a dotted key reference (e.g. key1.key2.key3)
+
+    >>> set(None, {'a':{'b':'c','d':{'e':'f'}},'g':'h'})
+    >>> set('g', 'update at root')
+    >>> get('g')
+    'update at root'
+    >>> set('w', 'new at root')
+    >>> get('w')
+    'new at root'
+    >>> set('a.d.e', 'update nested')
+    >>> get('a.d.e')
+    'update nested'
+    >>> set('a.d.y', 'new nested')
+    >>> get('a.d.y')
+    'new nested'
+    >>> get('a.d')
+    {'e': 'update nested', 'y': 'new nested'}
+    >>> set('a.d', {'z': 'overwrite'}, True)
+    >>> get('a.d')
+    {'z': 'overwrite'}
+    >>> set('l', ['a', 'b'])
+    >>> set('l', ['c', 'd'])
+    >>> get('l')
+    ['a', 'b', 'c', 'd']
+    >>> set('l', ['e', 'f'], True)
+    >>> get('l')
+    ['e', 'f']
+
+    """
+    if dottedkey is None:
+        # Special case - merge directly on to configuration - for tests
+        d = value
+    else:
+        if overwrite:
+            # Clear value first
+            set(dottedkey)
+        keys = dottedkey.split('.')
+        keys.reverse()
+        d = value
+        for k in keys:
+            d = {k: d}
+    always_merger.merge(configuration, d)

essmsync/core.py

+"""
+Core operations not tied to a specific service's API
+
+"""
+import logging
+import json
+import yaml
+
+from . import config
+
+LOG = logging.getLogger(__name__)
+
+
+# Argument helper functions
+
+def bool_arg(value):
+    """
+    As args are always parsed to strings, need to convert to a boolean.
+
+    """
+    if value is None or isinstance(value, bool):
+        return value
+    if isinstance(value, int):
+        return bool(int)
+    if value.isnumeric():
+        return bool(int(value))
+    return value.lower() != "false"
+
+
+def int_arg(value):
+    """
+    As args are always parsed to strings, need to convert to an integer.
+    Return a None if already or not convertable.
+
+    """
+    if value is None or isinstance(value, int):
+        return value
+    if isinstance(value, bool) or value.isnumeric():
+        return int(value)
+    return None
+
+
+# Core operations
+
+def op_test(*args):
+    """
+    Test operation (for testing)
+
+    """
+    if len(args) > 0:
+        print(f"Test called with {args}")
+    else:
+        print("Test called without arguments")
+
+
+def op_dump(key, keep_parents=True):
+    """
+    Shortcut to op_dump_yaml
+
+    """
+    op_dump_yaml(key, keep_parents)
+
+
+def op_dump_yaml(key, keep_parents=True):
+    """
+    Dump key of configuration as yaml, optionally maintaining parents of key.
+    By default, parents are maintained to help reusing output as extra
+    configuration to be included on future call.
+
+    >>> config.set(None, {'dy':{'a':'b','l':[{'id':1},{'id':2}]}})
+    >>> op_dump_yaml('dy.l')
+    dy:
+      l:
+      - id: 1
+      - id: 2
+    <BLANKLINE>
+    >>> op_dump_yaml('dy.l', False)
+    - id: 1
+    - id: 2
+    <BLANKLINE>
+    """
+    dump = config.get(key)
+    if bool_arg(keep_parents):
+        keys = key.split('.')
+        keys.reverse()
+        for k in keys:
+            dump = {k: dump}
+    print(yaml.dump(dump, sort_keys=True))
+
+
+def op_dump_json(key, keep_parents=False, indent=None):
+    """
+    Dump key of configuration as json, optionally maintaining parents of key
+    By default, parents are not maintained as a typical json dump is to get
+    just the specific object.
+    Indentation can be provided to 'prettify' the result using specified number
+    of spaces.
+
+    >>> config.set(None, {'dj':{'a':'b','l':[{'id':1},{'id':2}]}})
+    >>> op_dump_json('dj.l')
+    [{"id": 1}, {"id": 2}]
+    >>> op_dump_json('dj.l', True)
+    {"dj": {"l": [{"id": 1}, {"id": 2}]}}
+    >>> op_dump_json('dj.l', False, 2)
+    [
+      {
+        "id": 1
+      },
+      {
+        "id": 2
+      }
+    ]
+    """
+    dump = config.get(key)
+    if bool_arg(keep_parents):
+        keys = key.split('.')
+        keys.reverse()
+        for k in keys:
+            dump = {k: dump}
+    print(json.dumps(dump, sort_keys=True, indent=int_arg(indent)))

essmsync/exceptions.py

+"""
+All custom exceptions raised by tool
+
+"""
+
+
+class ConfigurationError(RuntimeError):
+    """
+    Base class for all configuration errors.
+
+    """
+
+
+class ConfigurationNotFound(ConfigurationError):
+    """
+    A configuration file could not be located.
+
+    """
+    def __init__(self):
+        return super().__init__('Configuration file missing')
+
+
+class ConfigurationInvalid(ConfigurationError):
+    """
+    Configuration is not valid
+
+    """
+    def __init__(self):
+        return super().__init__('Configuration is not valid')
+
+
+class AuthenticationError(RuntimeError):
+    """
+    Base class for all authentication errors.
+
+    """
+
+
+class APIError(RuntimeError):
+    """
+    Base class for all API errors.
+
+    """
+
+
+class TermTimeAPIError(APIError):
+    """
+    Base class for all TermTime API errors.
+
+    """
+
+
+class BookerAPIError(APIError):
+    """
+    Base class for all Booker API errors.
+
+    """
+
+
+class OperationInvalid(RuntimeError):
+    """
+    Operation specified is not valid
+
+    """

essmsync/sync.py

+"""
+Synchronisation actions
+
+"""
+import logging
+import re
+
+from . import config
+from . import core  # noqa: F401
+from . import termtime
+from . import booker
+from .exceptions import OperationInvalid
+
+LOG = logging.getLogger(__name__)
+
+# modules that contain valid operations
+VALID_MODULES = ['core', 'termtime', 'booker']
+
+
+def sync():
+    """
+    Perform sync actions
+
+    """
+    if config.get('read_only', True):
+        LOG.info('Performing synchronisation in READ ONLY mode.')
+    else:
+        LOG.info('Performing synchronisation in WRITE mode.')
+
+    # Parse and validate operations before calling any
+    # operations.validate returns tuple (func, args)
+    valid_ops = [validate_op(o) for o in config.get('operations')]
+    if len(valid_ops) == 0:
+        LOG.warning('No operations to perform')
+    else:
+        # Authenticate to each API
+        termtime.auth()
+        booker.auth()
+
+        # Perform operations
+        for (func, args, name) in valid_ops:
+            LOG.info(f'Performing: {name}')
+            func(*args)
+
+
+def validate_op(op):
+    """
+    Parse then validate operations (sync tasks/subtasks)
+
+    """
+    (module_name, func_name, args) = parse_op_string(op)
+    full_name = f'{module_name}:{func_name}:{args}'
+    if module_name == "" or func_name == "":
+        LOG.error(f'Invalid operation {op} ({full_name})')
+        raise OperationInvalid
+    if module_name not in VALID_MODULES:
+        LOG.error(f'Invalid operation module {op} ({full_name})')
+        raise OperationInvalid
+    # Get actual module from its name
+    module = globals()[module_name]
+    # Operation functions have 'op_' prefix
+    op_func_name = 'op_' + func_name
+    # Check function exists
+    if op_func_name not in dir(module):
+        LOG.error(f'Missing operation {op} ({full_name})')
+        raise OperationInvalid
+    # Actual function
+    func = getattr(module, op_func_name)
+
+    return (func, args, full_name)
+
+
+def parse_op_string(op):
+    """
+    Parses operation string in to components
+    Format: [module:]name[([arg[,arg]])]
+    If no module is provided 'core' is assumed
+
+    >>> parse_op_string('action')
+    ('core', 'action', [])
+    >>> parse_op_string('module:action')
+    ('module', 'action', [])
+    >>> parse_op_string('action(arg1)')
+    ('core', 'action', ['arg1'])
+    >>> parse_op_string('action(arg1,arg2)')
+    ('core', 'action', ['arg1', 'arg2'])
+    """
+    # defaults
+    module = 'core'
+    func_name = op
+    args = []
+    # match if arguments given
+    match = re.fullmatch(r'(.*)\((.*)\)', op)
+    if match is not None:
+        func_name = match.group(1)
+        args = [a.strip() for a in match.group(2).split(',')]
+    # match if module given
+    if ':' in func_name:
+        module, func_name = [a.strip() for a in func_name.split(':', 2)]
+
+    return (module, func_name, args)

essmsync/termtime.py

+"""
+Handle authentication and usage of TermTime API
+
+"""
+import logging
+
+from . import config
+from . import utils
+from .exceptions import TermTimeAPIError
+
+
+LOG = logging.getLogger(__name__)
+
+
+def tt_get(resource, **kwargs):
+    return utils.api_get(
+        config.get('termtime.api.url') + resource,
+        exception=TermTimeAPIError,
+        headers=tt_headers(),
+        **kwargs
+    )
+
+
+def tt_post(resource, **kwargs):
+    return utils.api_post(
+        config.get('termtime.api.url') + resource,
+        exception=TermTimeAPIError,
+        headers=tt_headers(),
+        **kwargs
+    )
+
+
+def tt_put(resource, **kwargs):
+    return utils.api_put(
+        config.get('termtime.api.url') + resource,
+        exception=TermTimeAPIError,
+        headers=tt_headers(),
+        **kwargs
+    )
+
+
+def tt_headers(form=False):
+    return {
+        'user-agent': config.USER_AGENT,
+        'Authorization': config.get('termtime.api.key'),
+    }
+
+
+def auth():
+    LOG.info('Authenticating TermTime API')
+
+    # No separate authentication step just use api key
+    (ping_resp, _) = tt_get('ping')
+    if ping_resp.get('ping') != 1:
+        LOG.error('TermTime API authentication failed')
+        raise TermTimeAPIError()
+
+    # If user and database provided check ping response matches
+    user = config.get('termtime.api.user')
+    db = config.get('termtime.api.database')
+    if user and db:
+        if ping_resp.get('user') == user and ping_resp.get('database') == db:
+            LOG.info('TermTime API user/database validated')
+        else:
+            LOG.error('TermTime API user/database mismatch')
+            raise TermTimeAPIError()
+
+
+# TermTime operations
+
+def op_get_rooms():
+    """
+    Get list of all rooms and put in `termtime.rooms`
+
+    """
+    (rooms, _) = tt_get('rooms')
+    if not isinstance(rooms, list):
+        LOG.error('TermTime rooms list is not a list')
+        raise TermTimeAPIError()
+    config.set('termtime.rooms', rooms, True)

essmsync/utils.py

+"""
+Common utility functions
+
+"""
+import requests
+import logging
+import json
+import time
+
+from .exceptions import APIError
+
+LOG = logging.getLogger(__name__)
+
+
+def api_get(url, **kwargs):
+    return api_via_method(requests.get, url, **kwargs)
+
+
+def api_post(url, **kwargs):
+    return api_via_method(requests.post, url, **kwargs)
+
+
+def api_put(url, **kwargs):
+    return api_via_method(requests.put, url, **kwargs)
+
+
+def api_via_method(method_func, url, exception=APIError, retries=2, **kwargs):
+    """
+    Uses `method_func` (the requests.get/post/put function) to call the API
+    endpoint.
+    If an HTTP, Connection, Timeout or JSON decode error occurs then the call
+    will be retried up to `retries` times.
+    If another error occurs or all retries have failed then an `exception` type
+    exception is raised.
+
+    """
+    try:
+        try:
+            r = method_func(url, **kwargs)
+            LOG.debug(f'Response [{r.status_code}]: {r.text}')
+            r.raise_for_status()
+            return (r.json(), r)
+        # Exceptions that can be retried
+        except (
+                json.decoder.JSONDecodeError,
+                requests.exceptions.ConnectionError,
+                requests.exceptions.Timeout,
+                requests.exceptions.HTTPError,
+               ) as e:
+            if retries > 0:
+                LOG.warning(f'Retrying: {e}')
+                # Wait 5 seconds before retrying
+                time.sleep(5)
+                return api_via_method(
+                    method_func, url, exception=exception, retries=retries-1, **kwargs
+                )
+            # Already retried enough times
+            raise
+    except requests.exceptions.RequestException as e:
+        LOG.error(f'Request {url} Failed: {e}')
+        raise exception()

requirements.txt

+docopt
+PyYAML
+deepmerge
+requests

secrets-example.yaml

+# Example ESSM sync configuration for secrets
+#
+# Multiple configuration files can be specified and will be 'deep-merged'
+termtime:
+  api:
+    key: "APIKEY"
+    # These are not needed for authentication but are validated against ping
+    # endpoint to confirm that correct key is being used, if provided
+    user: "essm-user"
+    database: "CamDatabase"
+
+booker:
+  api:
+    # if access_token is not given then a new token is retrieved from the token
+    # endpoint every run
+    access_token: ""
+    username: "essm-user@domain"
+    password: "BOOKER_PASSWORD"