FAQ | This is a LIVE service | Changelog

Commit 520d5a55 authored by Robin Goodall's avatar Robin Goodall 💬
Browse files

Check membership with Lookup and update

parent 89502635
Pipeline #176678 passed with stages
in 3 minutes and 38 seconds
......@@ -6,7 +6,7 @@ Usage:
lookupsync student-inst-members --gateway-client-id=CLIENT_ID --lookup-username=USERNAME
( --gateway-client-secret=CLIENT_SECRET | --gateway-client-secret-from=PATH )
( --lookup-password=PASSWORD | --lookup-password-from=PATH )
[--quiet] [--debug] [--really-do-this]
[--quiet] [--debug] [--lookup-test] [--really-do-this]
-h, --help Show a brief usage summary.
......@@ -28,6 +28,8 @@ Options:
Leading and trailing whitespace is trimmed. This is preferable
to passing secrets on the command line.
--lookup-test Use Lookup test instance instead of production
--really-do-this Actually attempt to update lookup group memberships, otherwise
just output what would have been changed.
......@@ -48,6 +50,7 @@ import pydantic
import requests_oauthlib
from identitylib.identifiers import Identifier, IdentifierSchemes
import ibisclient
API_GATEWAY_TOKEN_URL = 'https://api.apps.cam.ac.uk/oauth2/v1/token'
STUDENT_API_ROOT = 'https://api.apps.cam.ac.uk/university-student/v1alpha2/'
......@@ -92,8 +95,6 @@ def _student_inst_members(opts: dict, dry_run: bool):
Synchronise institutional memberships for students.
# TODO: This is incomplete.
session = _create_api_gateway_session(opts)
# Fetch institutional mapping and pre-compute a dict mapping Student Records Institution ids
......@@ -117,17 +118,13 @@ def _student_inst_members(opts: dict, dry_run: bool):
if len(inst_map) == 0:
raise RuntimeError('Failed to fetch any institutional mappings')
# TODO: map from person.v1.student-records.university.identifiers.cam.ac.uk to CRSid via USN
# identifier scheme in Lookup.
crsids_by_usn = {}
# A map from (institution, career) tuples to sets of students within that institution with
# status matching career. Institution identifiers are Lookup instids and student identifiers
# are CRSids
# A map from Lookup group name to sets of students within that institution with status
# matching career. Group names are formed from the Lookup instids and student identifiers
# are USNs
# Note that since students can be members of more than one institution, the sum of the lengths
# of each sets may not equal the length of the union of all of the sets.
students_by_inst = {}
students_by_group = {}
# Capture ignored affiliations and careers
ignored_affiliations = set()
......@@ -138,15 +135,14 @@ def _student_inst_members(opts: dict, dry_run: bool):
for s in _fetch_all_students(session):
# Add student to any institutions they are affiliated with.
for a in s.affiliations:
# Validate and normalise scheme
# Validate and normalise scheme
a.scheme = IdentifierSchemes.from_string(a.scheme, find_by_alias=True)
aff_scheme = IdentifierSchemes.from_string(a.scheme, find_by_alias=True)
except ValueError as e:
# Ignore non college/departmental affiliations.
if a.scheme != IdentifierSchemes.STUDENT_INSTITUTION:
if aff_scheme != IdentifierSchemes.STUDENT_INSTITUTION:
# Ignore expired or yet to be affiliations.
......@@ -168,27 +164,21 @@ def _student_inst_members(opts: dict, dry_run: bool):
# Ensure there is a set of student ids for this institution career group.
student_ids = students_by_inst.setdefault((instid, career), set())
student_ids = students_by_group.setdefault(_group_name(instid, career), set())
for i in s.identifiers:
# Validate and normalise scheme
# Validate and normalise scheme
i.scheme = IdentifierSchemes.from_string(i.scheme, find_by_alias=True)
id_scheme = IdentifierSchemes.from_string(i.scheme, find_by_alias=True)
except ValueError as e:
LOG.warn(f'Identifier: {i.value} ({a.value}) - "{e}"')
# Only identifiers from Student Records Person scheme (USNs)
if i.scheme != IdentifierSchemes.USN:
# Map USN to CRSid
crsid = crsids_by_usn.get(i.value)
if crsid is None:
LOG.warn(f'No CRSid for USN: {i.value} ({s.surname})')
if id_scheme != IdentifierSchemes.USN:
# Add USN to group
# Report ignored values (possibly missing from inst mapping or career mapping above)
if ignored_affiliations:
......@@ -201,24 +191,78 @@ def _student_inst_members(opts: dict, dry_run: bool):
LOG.info(f'- {a}')
# Sanity check
if len(students_by_inst) == 0:
if len(students_by_group) == 0:
raise RuntimeError('Failed to fetch any student records')
# Log some stats
LOG.info('Fetched record(s) for %s (institution, career) groups.', len(students_by_inst))
for (inst_id, career), students in students_by_inst.items():
LOG.info('Institution "%s" (%s) has %s student(s).', inst_id, career, len(students))
LOG.info('Total affiliation count: %s', sum(len(s) for s in students_by_inst.values()))
LOG.info('Fetched record(s) for %s groups.', len(students_by_group))
LOG.info('Total affiliation count: %s', sum(len(s) for s in students_by_group.values()))
all_students = set()
for s in students_by_inst.values():
for s in students_by_group.values():
all_students |= s
LOG.info('Total student count: %s', len(all_students))
# TODO: calculate changes
# TODO: limits / caps on changes
# TODO: actually perform synchronisation here.
ibis_conn = _create_lookup_connection(opts)
ibis_group_methods = ibisclient.GroupMethods(ibis_conn)
# calculate changes
missing_groups = set()
group_changes = dict()
for group, students in sorted(students_by_group.items()):
# Check that group exists
if ibis_group_methods.getGroup(group) is None:
LOG.info('Group "%s" should have %s student(s):', group, len(students))
members: List[ibisclient.IbisPerson] = ibis_group_methods.getDirectMembers(
group, 'all_identifiers')
LOG.info('- Lookup has %s member(s)', len(members))
group_usns = {
for person in members if person.identifiers is not None
for id in person.identifiers if id.scheme == 'usn'
'- with %s USNs%s', len(group_usns),
' - mismatch with membership' if len(group_usns) != len(members) else ''
to_add = students - group_usns
LOG.info('- %s need adding', len(to_add))
to_remove = group_usns - students
LOG.info('- %s need removing', len(to_remove))
if to_add | to_remove:
group_changes[group] = {'add': to_add, 'remove': to_remove}
if missing_groups:
# Shame we cannot automatically do this through API
LOG.info('Groups that need creating:')
for group in sorted(missing_groups):
LOG.info(f'- {group}')
LOG.info('%s group(s) need changes', len(group_changes))
# Make changes to Lookup groups
for group, changes in group_changes.items():
LOG.info('Updating %s:', group)
for usn in changes['add']:
LOG.info('- adding usn/%s', usn)
for usn in changes['remove']:
LOG.info('- removing usn/%s', usn)
if dry_run:
LOG.info('- skipping update in dry-run mode')
[f'usn/{usn}' for usn in changes['add']],
[f'usn/{usn}' for usn in changes['remove']],
'SIS Synchronisation',
return '{ "status": "ok" }', {'Content-Type': 'application/json'}
......@@ -279,6 +323,30 @@ def _create_api_gateway_session(opts: dict) -> requests_oauthlib.OAuth2Session:
return oauth
def _create_lookup_connection(opts: dict) -> ibisclient.IbisClientConnection:
Create an IbisClientConnection to either production or test Lookup API service, authenticated
with username and password from commandline or file.
username = opts['--lookup-username']
password = opts.get('--lookup-password')
password_from = opts.get('--lookup-password-from')
if password_from is not None:
with open(password_from) as fobj:
password = fobj.read().strip()
if opts['--lookup-test']:
LOG.info('Using test instance of Lookup')
ibis_conn = ibisclient.createTestConnection()
ibis_conn = ibisclient.createConnection()
return ibis_conn
def _fetch_all_students(
session: requests_oauthlib.OAuth2Session) -> Generator[List[Student], None, None]:
......@@ -294,3 +362,11 @@ def _fetch_all_students(
for s in data.get('results', []):
yield Student.parse_obj(s)
next_url = data.get('next')
def _group_name(instid: str, career: str):
Create Lookup group name from institution id and academic career mapping
return f'{instid.lower()}-sis-{career}'
......@@ -3,3 +3,7 @@ requests-oauthlib>=1.3.1,<2.0
# Until we are able to use group authentication through API Gateway, we need to
# use Lookup API directly
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment