FAQ | This is a LIVE service | Changelog

Commit 79dee0db authored by Robin Goodall's avatar Robin Goodall 💬
Browse files

Create missing groups before adding members

parent 051d1bff
......@@ -6,7 +6,7 @@ Usage:
lookupsync student-inst-members --gateway-client-id=CLIENT_ID --lookup-username=USERNAME
( --gateway-client-secret=CLIENT_SECRET | --gateway-client-secret-from=PATH )
( --lookup-password=PASSWORD | --lookup-password-from=PATH )
[--quiet] [--debug] [--lookup-test] [--really-do-this]
[--quiet] [--debug] [--lookup-test | --lookup-local] [--really-do-this]
-h, --help Show a brief usage summary.
......@@ -29,6 +29,7 @@ Options:
to passing secrets on the command line.
--lookup-test Use Lookup test instance instead of production
--lookup-local Use local instance of Lookup (developers only)
--really-do-this Actually attempt to update lookup group memberships, otherwise
just output what would have been changed.
......@@ -47,7 +48,9 @@ import ibisclient
from .api_gateway import create_api_gateway_session
from .inst_mapping import fetch_inst_mapping
from .student_api import get_students_by_group
from .lookup import create_lookup_connection, compare_with_lookup_groups, update_lookup_groups
from .lookup import (
create_lookup_connection, compare_with_lookup_groups,
update_lookup_groups, create_lookup_groups, strip_groups_missing_insts)
STUDENT_API_ROOT = 'https://api.apps.cam.ac.uk/university-student/v1alpha2/'
INST_MAPPING_API_ROOT = 'https://api.apps.cam.ac.uk/institutions/mapping/v1/'
......@@ -124,10 +127,17 @@ def _student_inst_members(opts: dict, dry_run: bool):
ibis_group_methods, students_by_group)
if missing_groups:
# Shame we cannot automatically do this through API
LOG.info('Groups that need creating:')
for group in sorted(missing_groups):
LOG.info(f'- {group}')
# Create groups that couldn't be found
ibis_inst_methods = ibisclient.InstitutionMethods(ibis_conn)
missing_insts = create_lookup_groups(ibis_inst_methods, missing_groups, dry_run)
if missing_insts:
LOG.info('%s institution(s) not found', len(missing_insts))
# Strip group changes for institutions that couldn't be found
previous_count = len(group_changes)
group_changes = strip_groups_missing_insts(group_changes, missing_insts)
if previous_count != len(group_changes):
LOG.info('%s groups(s) ignored as no matching institution found',
previous_count - len(group_changes))
# Make changes to Lookup groups
update_lookup_groups(ibis_group_methods, group_changes, dry_run)
......@@ -9,6 +9,19 @@ LOG = logging.getLogger(os.path.basename(sys.argv[0]))
# Convenient type definition for group changes dict
GroupChanges = Dict[str, Dict[str, Set[str]]]
# Make group 'career' suffix to group title suffix
'ug': 'Undergraduates',
'pg': 'Postgraduates',
# Fixed group description of created groups
"Group synchronised with student affiliations in the University's Student Information System."
# Transaction comments for group creation and update
def create_lookup_connection(opts: dict) -> ibisclient.IbisClientConnection:
......@@ -23,7 +36,10 @@ def create_lookup_connection(opts: dict) -> ibisclient.IbisClientConnection:
with open(password_from) as fobj:
password = fobj.read().strip()
if opts['--lookup-test']:
if opts['--lookup-local']:
LOG.info('Using local dev instance of Lookup')
ibis_conn = ibisclient.createLocalConnection()
elif opts['--lookup-test']:
LOG.info('Using test instance of Lookup')
ibis_conn = ibisclient.createTestConnection()
......@@ -63,14 +79,19 @@ def compare_with_lookup_groups(
# Check that group exists
if ibis_group_methods.getGroup(group) is None:
# Will want to add everyone after creating the group
group_changes[group] = {'add': students, 'remove': set()}
LOG.info('Group "%s" needs creating')
LOG.info('Group "%s" should have %s student(s):', group, len(students))
# Get lookup direct membership
members: List[ibisclient.IbisPerson] = ibis_group_methods.getDirectMembers(
group, 'all_identifiers')
LOG.info('- Lookup has %s member(s)', len(members))
# Get set of USNs from membership
group_usns = {
for person in members if person.identifiers is not None
......@@ -81,6 +102,7 @@ def compare_with_lookup_groups(
' - mismatch with membership' if len(group_usns) != len(members) else ''
# Determine who to add and/or remove
to_add = students - group_usns
LOG.info('- %s need adding', len(to_add))
to_remove = group_usns - students
......@@ -89,13 +111,15 @@ def compare_with_lookup_groups(
if to_add | to_remove:
group_changes[group] = {'add': to_add, 'remove': to_remove}
LOG.info('%s group(s) need creating', len(missing_groups))
LOG.info('%s group(s) need changes', len(group_changes))
return (missing_groups, group_changes)
def update_lookup_groups(
ibis_group_methods: ibisclient.GroupMethods,
group_changes: GroupChanges, dry_run: bool = True):
group_changes: GroupChanges,
dry_run: bool = True):
Log and update (if not a dry-run) group memberships
......@@ -113,5 +137,72 @@ def update_lookup_groups(
[f'usn/{usn}' for usn in changes['add']],
[f'usn/{usn}' for usn in changes['remove']],
'SIS Synchronisation',
def create_lookup_groups(
ibis_inst_methods: ibisclient.InstitutionMethods,
groups_to_create: Set[str],
dry_run: bool = True) -> Set[str]:
Log and create (if not a dry-run) lookup groups with names, titles, descriptions
and the current lookup account as the only manager.
Return a set of institution ids for institutions that couldn't be found.
# Get current authenticated lookup account
managed_by = ibis_inst_methods.conn.username
# Cache institution details so we can name, title and describe groups appropriately
institutions: Dict[str, ibisclient.IbisInstitution] = {}
# Compile a set of institutions we couldn't find
missing_insts = set()
for group in sorted(groups_to_create):
# split group name into instid and career (ug or pg)
(instid, _, career) = group.split('-')
instid = instid.upper()
if instid not in institutions:
# Make sure institution exists
inst = ibis_inst_methods.getInst(instid)
if inst is None:
LOG.warning('Institution "%s" not found for group "%s"', instid, group)
institutions[instid] = inst
inst_name = institutions[instid].name
group_title = f'{inst_name} - SIS - {GROUP_TITLE_MAPPING[career]}'
LOG.info('Creating group "%s"', group)
LOG.info('- in institution "%s"', instid)
LOG.info('- with title "%s"', group_title)
if dry_run:
LOG.info('- skipping creation in dry-run mode')
instid, group, group_title,
return missing_insts
def strip_groups_missing_insts(
group_changes: GroupChanges, missing_insts: Set[str]) -> GroupChanges:
Remove group changes that would belong to institutions that couldn't be found.
>>> strip_groups_missing_insts(
... {
... 'abc-sis-pg': {'add': {'123'}, 'remove': {'456'}},
... 'def-sis-ug': {'add': {'234'}, 'remove': {'567'}},
... },
... {'DEF', 'GHI'}
... )
{'abc-sis-pg': {'add': {'123'}, 'remove': {'456'}}}
return {
group: changes for group, changes in group_changes.items()
if group.split('-')[0].upper() not in missing_insts
......@@ -3,7 +3,4 @@ requests-oauthlib>=1.3.1,<2.0
# Until we are able to use group authentication through API Gateway, we need to
# use Lookup API directly
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment