FAQ | This is a LIVE service | Changelog

Commit 06bd4cff authored by Dave Hart's avatar Dave Hart

Merge branch 'comparison-rooms' into 'master'

Comparison operations for depts, sites, buildings, floors and rooms

Closes #16 and #11

See merge request !7
parents 731acb17 21253f26
Pipeline #40564 passed with stages
in 3 minutes and 57 seconds
.git
venv/
.venv/
*.pyc
__pycache__/
......
# Editorconfig file for cross-platform configuration of editors.
root=true
[*.py]
max_line_length=99
[*.{yml,yaml}]
indent_style=space
indent_size=2
[*.md]
indent_style=space
indent_size=2
[Dockerfile]
indent_style=space
indent_size=2
......@@ -16,3 +16,8 @@ include:
variables:
DOCUMENTATION_DISABLED: "1"
DAST_DISABLED: "1"
test:
artifacts:
reports:
cobertura: ./artefacts/**/coverage.xml
......@@ -3,12 +3,15 @@ FROM uisautomation/python:3.7-alpine
WORKDIR /usr/src/app
# Install specific requirements for the package.
ADD requirements.txt ./
RUN pip install tox && pip install -r requirements.txt
ADD ./requirements.txt ./requirements-tests.txt ./
RUN pip install --upgrade pip && \
pip install -r requirements.txt && \
pip install -r requirements-tests.txt && \
pip install tox
# Copy application source and install it. Use "-e" to avoid needlessly copying
# files into the site-packages directory.
ADD ./ ./
RUN pip install -e .
RUN pip install .
ENTRYPOINT ["essmsync"]
......@@ -54,6 +54,7 @@ See the output of `essmsync --help` for more information.
### Docker Usage
Running sync-tool directly from repository:
```bash
$ docker run --rm -ti -v $(pwd):/usr/src/app \
registry.gitlab.developers.cam.ac.uk/uis/devops/essm/sync-tool/master \
......@@ -61,6 +62,21 @@ $ docker run --rm -ti -v $(pwd):/usr/src/app \
test(inside,docker)
```
Alternatively, an image can be build locally:
```bash
$ cd /path/to/repo
$ docker build -t essmsync .
# Assuming configurations files in another location
$ cd /path/to/configuration
$ docker run --rm -ti -v $(pwd):/usr/src/app essmsync \
-c termtime-test.yaml -c booker-staging.yaml \
test(local,docker)
```
Tests can then be run by overriding the entrypoint with `tox`:
```bash
$ docker run --rm -ti --entrypoint=tox essmsync
```
## Operations
......
......@@ -199,8 +199,8 @@ def op_get_depts():
if state.get('booker.rooms') is None:
op_get_rooms()
# Get all (non -1) department OptimeIndexes for filtered rooms
dept_ois = [d['DepartmentId'] for d in state.get('booker.rooms')
if d.get('DepartmentId', -1) != -1]
dept_ois = {d['DepartmentId'] for d in state.get('booker.rooms')
if d.get('DepartmentId', -1) != -1}
if len(dept_ois) == 0:
LOG.error('No departments in Booker related to TermTime rooms')
raise BookerAPIError()
......@@ -251,8 +251,8 @@ def op_get_sites():
if state.get('booker.buildings') is None:
op_get_buildings()
# Get all (non -1) site OptimeIndexes for filtered buildings
site_ois = [b['Site'] for b in state.get('booker.buildings')
if b.get('Site', -1) != -1]
site_ois = {b['Site'] for b in state.get('booker.buildings')
if b.get('Site', -1) != -1}
if len(site_ois) == 0:
LOG.error('No sites in Booker related to TermTime rooms')
raise BookerAPIError()
......
"""
Operations that compare the results of TermTime and Booker API get operations
"""
import logging
from . import state
from . import booker
from . import termtime
from .exceptions import ComparisonError
LOG = logging.getLogger(__name__)
def _op_compare_b_to_tt(data_type, booker_state, termtime_state, mapping, code='code'):
"""
Generic comparison between Booker and TermTime, creating lists of necessary
creations and updated in TermTime, and putting them in comparison state.
:param data_type: type of data being compared as resulting state sub-key (e.g. sites)
:type data_type: str
:param booker_state: booker state key to use in comparison (e.g. booker.sites)
:type booker_state: str
:param termtime_state: termtime state key to use in comparison (e.g. termtime.campuses)
:type termtime_state: str
:param mapping: function called passed each booker entity that returns a dict containing
- 'expected' - expected TermTime keys and values
- 'update' - keys and values need to create or update TermTime
Returns None - if entry is invalid
:type comparitor: callable[[dict], dict]
:param code: termtime field used for code (usually 'code', sometimes 'c'), defaults to 'code'
:type code: str, optional
"""
# TermTime entries by code (sometimes 'c'!)
tt_by_code = {d[code]: d for d in state.get(termtime_state)}
# Results of comparison
results = {
'invalid': [],
'match': [],
'create': [],
'update': [],
'delete': [], # in TermTime but not Booker
}
# Track matched TermTime entry codes
matched_codes = []
# Check each booker entry
LOG.info(f'Comparing Booker {data_type} with TermTime')
for d in state.get(booker_state):
id = d.get('Id')
# Booker data must have at least Id
if id is None:
desc = d.get('Description', 'Missing Description')
LOG.warning(f"Booker {data_type} missing Id - '{desc}'")
results['invalid'].append(d)
continue
# Get expected TermTime data and create/update data
tt_mapping = mapping(d)
if tt_mapping is None:
# Booker entry invalid
results['invalid'].append(d)
continue
# Set update mapping code (either 'c' or 'code')
tt_mapping['update'][code] = id
# Check existance in TermTime and match
if id in tt_by_code:
# flag as exists to aid finding extra TermTime entries later
matched_codes.append(id)
if _compare_tt(tt_mapping['expected'], tt_by_code[id], data_type, id):
# comparison successful, add expected (with code) to 'match' results
tt_mapping['expected'][code] = id
results['match'].append(tt_mapping['expected'])
else:
# comparison failed, add needed update to 'update' results
results['update'].append(tt_mapping['update'])
else:
# id not found so need to create
results['create'].append(tt_mapping['update'])
# Find those in TermTime but not Booker (ignoring many with code of None or 'None'!)
results['delete'] = [
d for c, d in tt_by_code.items()
if c is not None and c != 'None' and c not in matched_codes
]
# Log counts
for k in results:
LOG.info(f"...{k}: '{len(results[k])}'")
# Put results of comparison in state
state.set(f'comparison.{data_type}', results)
def op_compare_depts():
"""
Compare departments between Booker and TermTime, and creating lists of necessary
creations and updates needed and putting them in `comparison.depts`
"""
# Get filtered depts list from Booker, if necessary
if state.get('booker.depts') is None:
booker.op_get_depts()
# Get full depts list from TermTime, if necessary
if state.get('termtime.depts') is None:
termtime.op_get_depts()
# Perform comparison - just Description to 'n' (depts use 'c' for code)
_op_compare_b_to_tt('depts', 'booker.depts', 'termtime.depts', _mapping_n, 'c')
def op_compare_sites():
"""
Compare sites/campuses between Booker and TermTime, and creating lists of necessary
creations and updates needed and putting them in `comparison.sites`
"""
# Get filtered sites list from Booker, if necessary
if state.get('booker.sites') is None:
booker.op_get_sites()
# Get full campuses list from TermTime, if necessary
if state.get('termtime.campuses') is None:
termtime.op_get_campuses()
# Perform comparison - just Description to 'n' (sites use 'code' for code)
_op_compare_b_to_tt('sites', 'booker.sites', 'termtime.campuses', _mapping_n)
def op_compare_buildings():
"""
Compare buildings between Booker and TermTime, and creating lists of necessary
creations and updates needed and putting them in `comparison.buildings`
"""
# Get filtered buildings list from Booker, if necessary
if state.get('booker.buildings') is None:
booker.op_get_buildings()
# Get full buildings list from TermTime, if necessary
if state.get('termtime.buildings') is None:
termtime.op_get_buildings()
# Also need booker sites to match site OptimeIndex to site Id
if state.get('booker.sites') is None:
booker.op_get_sites()
sites_by_oi = {s['OptimeIndex']: s for s in state.get('booker.sites')}
def mapping(booker):
"""
In addition to comparing 'Description' to 'n', also check that site's
Description matches campus name ('cn').
TermTime get buildings returns 'cn' for campus name but update uses 'camp'
for campus name or code (better to use code)
"""
# Common 'Description' to 'n' mapping
m = _mapping_n(booker)
# Validate Site (OptimeIndex) exists
site_oi = booker.get('Site')
if site_oi is None:
LOG.warning(f"Booker building missing Site - '{booker.get('Id')}'")
return None
# Lookup site to get name and id
site = sites_by_oi.get(site_oi)
if site is None:
LOG.warning(f"Booker building has missing Site '{site_oi}' - '{booker.get('Id')}'")
return None
# Add Campuses Name (for comparison)
m['expected']['cn'] = site.get('Description')
# Add Campus Id (for update/create)
m['update']['camp'] = site.get('Id')
return m
# Perform comparison
_op_compare_b_to_tt('buildings', 'booker.buildings', 'termtime.buildings', mapping)
def op_compare_rooms():
"""
Compare rooms between Booker and TermTime, and creating lists of necessary
creations and updates needed and putting them in `comparison.rooms`
"""
# Get filtered rooms list from Booker, if necessary
if state.get('booker.rooms') is None:
booker.op_get_rooms()
# Get full rooms list from TermTime, if necessary
if state.get('termtime.rooms') is None:
termtime.op_get_rooms()
# Also need booker sites to match building.site OptimeIndex to site Id
if state.get('booker.sites') is None:
booker.op_get_sites()
sites_by_oi = {s['OptimeIndex']: s for s in state.get('booker.sites')}
# Also need booker departments to match DepartmentId OptimeIndex to department's Id
if state.get('booker.depts') is None:
booker.op_get_depts()
depts_by_oi = {d['OptimeIndex']: d for d in state.get('booker.depts')}
def mapping(booker):
"""
In addition to comparing 'Description' to 'n', also check that building
code (f.bc), department code (d.c) and floor name (f.n) match.
Create/update entry also needs floor, buildings and campus names (f, b, cam).
"""
# Common 'Description' to 'n' mapping
m = _mapping_n(booker)
# Validate Building dict exists
b = booker.get('Building')
if b is None or not isinstance(b, dict):
LOG.warning(f"Booker room missing/invalid Building - '{booker.get('Id')}'")
return None
# Validate Building.Id and Description exists
b_id = b.get('Id')
if b_id is None:
LOG.warning(f"Booker room missing Building.Id - '{booker.get('Id')}'")
return None
b_name = b.get('Description')
if b_name is None:
LOG.warning(f"Booker room missing Building.Description - '{booker.get('Id')}'")
return None
# Validate Building.Site (OptimeIndex) exists
site_oi = b.get('Site')
if site_oi is None:
LOG.warning(f"Booker room missing Building.Site - '{booker.get('Id')}'")
return None
# Lookup site to get name and id
site = sites_by_oi.get(site_oi)
if site is None:
LOG.warning(f"Booker room has missing Site '{site_oi}' - '{booker.get('Id')}'")
return None
# Validate DepartmentId (OptimeIndex) exists
dept_oi = booker.get('DepartmentId')
if dept_oi is None:
LOG.warning(f"Booker room missing DepartmentId - '{booker.get('Id')}'")
return None
# Lookup site to get name and id
dept = depts_by_oi.get(dept_oi)
if dept is None:
LOG.warning(f"Booker room has missing Department '{dept_oi}' - '{booker.get('Id')}'")
return None
# Validate Floor name exists
floor = booker.get('Floor')
if floor is None:
LOG.warning(f"Booker room missing Floor - '{booker.get('Id')}'")
return None
# Add building code, floor name and department code (for comparison)
m['expected']['f'] = {'bc': b_id, 'n': floor}
m['expected']['d'] = {'c': dept.get('Id')}
# Add floor, building, campus names and department code (for update)
m['update']['f'] = floor
m['update']['b'] = b_name
m['update']['cam'] = site.get('Description')
m['update']['dn'] = dept.get('Id')
return m
# Perform comparison - rooms use 'c' for code
_op_compare_b_to_tt('rooms', 'booker.rooms', 'termtime.rooms', mapping, 'c')
def op_compare_floors():
"""
Compare floors between Booker and TermTime, and creating lists of necessary
creations and updates needed and putting them in `comparison.floors`
Booker floors are just strings on a room. The booker.get_floors operation
constructs floors with 'Name', 'Building.Description' and 'Building.Site'
(OptimeIndex that needs converting to a site Description). TermTime's floor
'code's have no matching 'Id' in Booker, so a comparison by matching these
is not possible. This comparison operation differs from by matching on multiple
values.
"""
# Get constructed floors list from Booker, if necessary
if state.get('booker.floors') is None:
booker.op_get_floors()
# Get full floors list from TermTime, if necessary
if state.get('termtime.floors') is None:
termtime.op_get_floors()
# Also need booker sites to match site OptimeIndex to site Description
if state.get('booker.sites') is None:
booker.op_get_sites()
sites_by_oi = {s['OptimeIndex']: s for s in state.get('booker.sites')}
# TermTime entries as tuples (campus name, building name, floor name)
tt_by_code = {f.get('code'): f for f in state.get('termtime.floors')}
# Results of comparison
results = {
'invalid': [],
'match': [],
'create': [],
'update': [], # always empty as one-to-one comparisons not possible
'delete': [], # in TermTime but not Booker
}
# Track matched TermTime entry codes
matched_codes = []
# Check each booker entry
LOG.info('Comparing Booker floors with TermTime')
for floor in state.get('booker.floors'):
# Validate floor name exists (and isn't blank)
name = floor.get('Name')
if name is None or name == '':
LOG.warning(f"Booker floor missing/empty Name - '{floor.get('Id')}'")
results['invalid'].append(floor)
continue
# Validate Building dict exists
b = floor.get('Building')
if b is None or not isinstance(b, dict):
LOG.warning(f"Booker floor missing/invalid Building - '{floor.get('Id')}'")
results['invalid'].append(floor)
continue
# Validate Building.Site (OptimeIndex) exists
site_oi = b.get('Site')
if site_oi is None:
LOG.warning(f"Booker floor missing Building.Site - '{floor.get('Id')}'")
results['invalid'].append(floor)
continue
# Lookup site to get name and id
site = sites_by_oi.get(site_oi)
if site is None:
LOG.warning(f"Booker floor has missing Site '{site_oi}' - '{floor.get('Id')}'")
results['invalid'].append(floor)
continue
# Create matching tuple to search for
expected = {
'cn': site.get('Description'),
'bn': b.get('Description'),
'n': name,
}
def compare_floors(expected, actual):
for field in ['cn', 'bn', 'n']:
if expected.get(field) != actual.get(field):
return False
return True
# Create list of matching TermTime floors
matched = [c for c, f in tt_by_code.items()
if compare_floors(expected, f)]
if len(matched) == 0:
LOG.info(f"Floor {floor.get('Id')} not found")
results['create'].append({
'f': name,
'b': b.get('Id'),
'c': site.get('Id'),
})
elif len(matched) > 1:
# Couldn't find a single match - not good
LOG.warning(f"Booker floor matches {len(matched)} TermTime floors "
f"- '{floor.get('Id')}'")
results['invalid'].append(floor)
continue
else:
# Successfully matched
tt_floor = tt_by_code.get(matched[0])
results['match'].append(tt_floor)
matched_codes.append(matched[0])
# Find those in TermTime but not Booker
results['delete'] = [d for c, d in tt_by_code.items() if c not in matched_codes]
# Log counts
for k in results:
LOG.info(f"...{k}: '{len(results[k])}'")
# Put results of comparison in state
state.set('comparison.floors', results)
def _compare_tt(expected, actual, data_type, id):
"""
Compares all the keys (and sub-keys) in 'expected' dict with the values in
'actual' dict. Returns False if any don't match.
>>> # single level matching returns True
>>> _compare_tt({'a':1, 'b':'two'}, {'a':1, 'b':'two'}, 'test', 1)
True
>>> # single level mismatched returns False
>>> _compare_tt({'a':1, 'b':'two'}, {'a':1, 'b':'three'}, 'test', 1)
False
>>> # nested matching returns True
>>> _compare_tt({'a':{'b':'two'}}, {'a':{'b':'two'}}, 'test', 1)
True
>>> # nested mismatched returns False
>>> _compare_tt({'a':{'b':'two'}}, {'a':{'b':'three'}}, 'test', 1)
False
>>> # missing key in actual raises ComparisonError
>>> _compare_tt({'a':1, 'b':'two'}, {'a':1, 'c':'bad'}, 'test', 1)
Traceback (most recent call last):
...
essmsync.exceptions.ComparisonError: Missing expected key
>>> # dict in expected but not in actual raises ComparisonError
>>> _compare_tt({'a':{'b':'two'}}, {'a':1, 'b':'bad'}, 'test', 1)
Traceback (most recent call last):
...
essmsync.exceptions.ComparisonError: Expected dict
>>> # dict in actual but not in expected raises ComparisonError
>>> _compare_tt({'a':1}, {'a':{'b':'bad'}}, 'test', 1)
Traceback (most recent call last):
...
essmsync.exceptions.ComparisonError: Unexpected dict
"""
success = True
for k, v in expected.items():
# Check for fatal missing and type mismatches before doing comparison
if k not in actual:
LOG.error(f"'{k}' missing from {data_type} '{id}'")
raise ComparisonError('Missing expected key')
elif isinstance(v, dict):
if not isinstance(actual[k], dict):
LOG.error(f"'{k}' expected to be dict in {data_type} '{id}'")
raise ComparisonError('Expected dict')
# Recursively, compare dicts
if not _compare_tt(v, actual[k], data_type, id):
success = False
elif isinstance(actual[k], dict):
LOG.error(f"'{k}' not expected to be dict in {data_type} '{id}'")
raise ComparisonError('Unexpected dict')
elif v != actual[k]:
LOG.info(f"'{k}' mismatch for {data_type} '{id}' ('{v}' != '{actual[k]}')")
success = False
return success
# Common mapping function
def _mapping_n(booker):
"""
Just need to compare 'Description' to 'n'
>>> _mapping_n({}) is None
True
>>> _mapping_n({'Description':'Hypothetical Building'})
{'expected': {'n': 'Hypothetical Building'}, 'update': {'n': 'Hypothetical Building'}}
>>> _mapping_n({'Description':' Unnecessarily Spaced Site '})
{'expected': {'n': 'Unnecessarily Spaced Site'}, 'update': {'n': 'Unnecessarily Spaced Site'}}
"""
desc = booker.get('Description')
if desc is None:
LOG.warning(f"Booker entry missing Description - '{booker.get('Id')}'")
return None
# Booker Descriptions appear to frequently have trailing spaces!
desc = desc.strip()
return {
'expected': {'n': desc},
'update': {'n': desc},
}
......@@ -16,15 +16,30 @@ LOG = logging.getLogger(__name__)
def bool_arg(value):
"""
As args are always parsed to strings, need to convert to a boolean.
Return a None if already or not convertable.
>>> bool_arg(0) or bool_arg("0")
False
>>> bool_arg(1) and bool_arg("1")
True
>>> bool_arg("true") and bool_arg("True")
True
>>> bool_arg("false") or bool_arg("False")
False
>>> bool_arg(None) is None
True
>>> bool_arg(['unconvertable']) is None
True
"""
if value is None or isinstance(value, bool):
return value
if isinstance(value, int):
return bool(int)
if value.isnumeric():
return bool(int(value))
return value.lower() != "false"
return bool(value)
if isinstance(value, str):
if value.isnumeric():
return bool(int(value))
return value.lower() != "false"
return None
def int_arg(value):
......@@ -32,6 +47,12 @@ def int_arg(value):
As args are always parsed to strings, need to convert to an integer.
Return a None if already or not convertable.
>>> int_arg(100) == int_arg("100") == 100
True
>>> int_arg(1) == int_arg("1") == int_arg(True) == 1
True
>>> int_arg(0) == int_arg("0") == int_arg(False) == 0
True
"""