FAQ | This is a LIVE service | Changelog

Commit f27b771c authored by Robin Goodall's avatar Robin Goodall 💬
Browse files

Filter USNs not recognised by Lookup

parent 7b28892a
Pipeline #208014 passed with stages
in 3 minutes and 40 seconds
......@@ -49,8 +49,9 @@ from .api_gateway import create_api_gateway_session
from .inst_mapping import fetch_inst_mapping
from .student_api import get_students_by_group
from .lookup import (
create_lookup_connection, compare_with_lookup_groups,
update_lookup_groups, create_lookup_groups, strip_groups_missing_insts)
create_lookup_connection, compare_with_lookup_groups, check_usns_in_lookup,
filter_usns_from_changes, create_lookup_groups, strip_groups_missing_insts,
STUDENT_API_ROOT = 'https://api.apps.cam.ac.uk/university-student/v1alpha2/'
INST_MAPPING_API_ROOT = 'https://api.apps.cam.ac.uk/institutions/mapping/v1/'
......@@ -121,14 +122,30 @@ def _student_inst_members(opts: dict, dry_run: bool):
ibis_conn = create_lookup_connection(opts)
ibis_group_methods = ibisclient.GroupMethods(ibis_conn)
ibis_person_methods = ibisclient.PersonMethods(ibis_conn)
ibis_inst_methods = ibisclient.InstitutionMethods(ibis_conn)
# Calculate changes and find missing groups
(missing_groups, group_changes) = compare_with_lookup_groups(
ibis_group_methods, students_by_group, student_names_by_id)
# Find USNs (that are being added) that are not recognised by lookup
all_usns_to_add = {
for _, changes in group_changes.items()
for usn in changes['add']
missing_usns = check_usns_in_lookup(ibis_person_methods, all_usns_to_add)
# Filter missing USNs from group changes
filter_usns_from_changes(group_changes, missing_usns, student_names_by_id)
# No need to create groups if they no longer have members to add
missing_groups = {
group for group in missing_groups if group_changes.get(group)['add']
if missing_groups:
# Create groups that couldn't be found
ibis_inst_methods = ibisclient.InstitutionMethods(ibis_conn)
missing_insts = create_lookup_groups(ibis_inst_methods, missing_groups, dry_run)
if missing_insts:
LOG.info('%s institution(s) not found', len(missing_insts))
from typing import Dict, List, Tuple
import logging
from typing import Dict, Generator, Iterable, List, Tuple
import os
import sys
import logging
import itertools
import ibisclient
from .types import (GroupChange, GroupChanges, GroupName, GroupSet,
InstIdSet, LookupInstId,
StudentMapping, StudentsByGroup)
StudentIdSet, StudentMapping, StudentsByGroup)
LOG = logging.getLogger(os.path.basename(sys.argv[0]))
......@@ -23,6 +25,9 @@ GROUP_DESCRIPTION = (
# Transaction comments for group creation and update
# Request lists of people from lookup in chunks to avoid overflowing querystring
def create_lookup_connection(opts: dict) -> ibisclient.IbisClientConnection:
......@@ -107,10 +112,10 @@ def compare_with_lookup_groups(
# Determine who to add and/or remove
to_add = students - group_usns
if to_add:
LOG.info('- %s need adding', len(to_add))
LOG.info('- %s need%s adding', len(to_add), 's' if len(to_add) == 1 else '')
to_remove = group_usns - students
if to_remove:
LOG.info('- %s need removing', len(to_remove))
LOG.info('- %s need%s removing', len(to_remove), 's' if len(to_add) == 1 else '')
# Add their names to our id to name mapping
id.value: person.visibleName
......@@ -126,6 +131,71 @@ def compare_with_lookup_groups(
return (missing_groups, group_changes)
def chunker(n: int, it: Iterable) -> Generator[Tuple[int, StudentIdSet], None, None]:
Yield chunks of size `n` from `it`. The last chunk may be smaller.
If `n` is not greater than the size of `it`, just yields the one chunk
if n <= 0:
yield 0, list(it)
args = [iter(it)] * n
chunks = itertools.zip_longest(*args, fillvalue=None)
# unfortunately we need to drop the Nones on the last element
for i, chunk in enumerate(chunks):
yield i, set(filter(None, chunk))
def check_usns_in_lookup(
ibis_person_methods: ibisclient.PersonMethods,
all_usns_to_add: StudentIdSet) -> StudentIdSet:
Check that the set of students to add have USNs recognised by Lookup.
Return a list of those that don't.
matched_usns: StudentIdSet = set()
LOG.info('Checking USNs exist in Lookup:')
for idx, usns in chunker(CHUNK_SIZE, all_usns_to_add):
people: List[ibisclient.IbisPerson] = ibis_person_methods.listPeople(
[f'usn/{usn}' for usn in usns],
usns_of_people: StudentIdSet = {
for person in people if person.identifiers is not None
for id in person.identifiers if id.scheme == 'usn'
LOG.info(f'- batch {idx} : missing {len(usns) - len(usns_of_people)} of {len(usns)} USNs')
matched_usns |= usns_of_people
return all_usns_to_add - matched_usns
def filter_usns_from_changes(
group_changes: GroupChanges, missing_usns: StudentIdSet,
student_names_by_id: StudentMapping
) -> None:
Filter USNs to add from the changes if they are in the missing USNs list.
Log which USNs are ignored for each group.
for group, changes in group_changes.items():
missing_in_group_add = {
usn for usn in changes['add'] if usn in missing_usns
if missing_in_group_add:
LOG.info('Ignoring in group %s:', group)
for usn in missing_in_group_add:
LOG.info('- usn/%s (%s)', usn, student_names_by_id.get(usn, 'Unknown'))
# remove ignored USNs from set to add
changes['add'] = {
usn for usn in changes['add'] if usn not in missing_in_group_add
def update_lookup_groups(
ibis_group_methods: ibisclient.GroupMethods,
group_changes: GroupChanges,
......@@ -136,6 +206,8 @@ def update_lookup_groups(
for group, changes in group_changes.items():
if not changes['add'] and not changes['remove']:
LOG.info('Updating %s:', group)
for usn in changes['add']:
LOG.info('- adding usn/%s (%s)', usn, student_names_by_id.get(usn, 'Unknown'))
......@@ -12,18 +12,19 @@ InstIdSet = Set[LookupInstId]
StudentId = str
StudentName = str
StudentMapping = Dict[StudentId, StudentName]
StudentIdSet = Set[StudentId]
# Group names as strings, and we use a mapping of group names to sets of student ids
GroupName = str
StudentsByGroup = Dict[GroupName, Set[StudentId]]
StudentsByGroup = Dict[GroupName, StudentIdSet]
GroupSet = Set[GroupName]
# A group change consists of a set of student ids to add and a set to remove
class GroupChange(TypedDict):
add: Set[StudentId]
remove: Set[StudentId]
add: StudentIdSet
remove: StudentIdSet
# A collection of group changes is a mapping of group name to changes
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment