From e7b59058b20db56ab7f648c572b3d55cd886cfd1 Mon Sep 17 00:00:00 2001
From: Dean Rasheed <dar17@cam.ac.uk>
Date: Thu, 13 Aug 2020 17:49:53 +0100
Subject: [PATCH] Synchronise Lookup institutions.
This causes Lookup institutions and their members to be synchronised
with groups in Google, in a similar way to Lookup groups. The unique
identifier of the group in Google is it's email address:
email = {instID.lower()}@{insts_domain}
where {insts_domain} is a configurable secondary domain (noting that
the local part of the email address is required to be lowercase,
whereas all instIDs from Lookup are uppercase).
---
configuration-example.yaml | 18 +++
gsuitesync/gapidomain.py | 4 +
gsuitesync/ldap.py | 63 +++++++++
gsuitesync/sync.py | 281 ++++++++++++++++++++++++-------------
4 files changed, 267 insertions(+), 99 deletions(-)
diff --git a/configuration-example.yaml b/configuration-example.yaml
index 31ce46c..0481973 100644
--- a/configuration-example.yaml
+++ b/configuration-example.yaml
@@ -150,6 +150,9 @@ ldap:
# LDAP search base for groups. Group filters are always relative to this.
group_search_base: 'ou=groups,o=example-corps,dc=example,dc=com'
+ # LDAP search base for institutions. Institution filters are always relative to this.
+ inst_search_base: 'ou=insts,o=example-corps,dc=example,dc=com'
+
# Filter to use to determine the "eligible" list of users. If a non-admin user
# is found on Google who isn't in this list, their account will be suspended.
eligible_user_filter: '(uid=*)'
@@ -158,6 +161,10 @@ ldap:
# found on Google that isn't in this list, it will be deleted.
eligible_group_filter: '(groupID=*)'
+ # Filter to use to determine the "eligible" list of institutions. If an
+ # institution is found on Google that isn't in this list, it will be deleted.
+ eligible_inst_filter: '(instID=*)'
+
# Filter to use to determine the "managed" list of users. If a user appears in
# this list who isn't in Google their account is created. If the user metadata
# for a user in this list changes, the change is propagated to Google. If
@@ -171,6 +178,13 @@ ldap:
# Default: null.
managed_group_filter: null
+ # Filter to use to determine the "managed" list of institutions. If an
+ # institution appears in this list that isn't in Google it is created. If the
+ # institution metadata or list of members for an institution in this list
+ # changes, the change is propagated to Google. If null, the value of
+ # "eligible_inst_filter" is used. Default: null.
+ managed_inst_filter: null
+
# Details about the Google Domain we're managing.
google_domain:
# Name of the domain.
@@ -185,3 +199,7 @@ google_domain:
# Secondary domain or domain alias for groups. If null, the value of "name"
# is used. Default: null
groups_domain: null
+
+ # Secondary domain or domain alias for institutions. If null, the value of
+ # "name" is used. Default: null
+ insts_domain: null
diff --git a/gsuitesync/gapidomain.py b/gsuitesync/gapidomain.py
index 9e7e1eb..21d0923 100644
--- a/gsuitesync/gapidomain.py
+++ b/gsuitesync/gapidomain.py
@@ -25,3 +25,7 @@ class Configuration(ConfigurationDataclassMixin):
# Secondary domain or domain alias for groups. If None, the value of "name" is used.
# Default: None
groups_domain: typing.Union[str, None] = None
+
+ # Secondary domain or domain alias for institutions. If None, the value of "name" is used.
+ # Default: None
+ insts_domain: typing.Union[str, None] = None
diff --git a/gsuitesync/ldap.py b/gsuitesync/ldap.py
index fa39881..62d8447 100644
--- a/gsuitesync/ldap.py
+++ b/gsuitesync/ldap.py
@@ -28,14 +28,20 @@ class Configuration(ConfigurationDataclassMixin):
group_search_base: str
+ inst_search_base: str
+
eligible_user_filter: str
eligible_group_filter: str
+ eligible_inst_filter: str
+
managed_user_filter: typing.Union[str, None] = None
managed_group_filter: typing.Union[str, None] = None
+ managed_inst_filter: typing.Union[str, None] = None
+
def get_eligible_uids(self):
"""
Return a set containing all uids who are eligible to have a Google account.
@@ -62,6 +68,19 @@ class Configuration(ConfigurationDataclassMixin):
)
}
+ def get_eligible_instIDs(self):
+ """
+ Return a set containing all instIDs that are eligible for Google.
+
+ """
+ return {
+ e['attributes']['instID'][0]
+ for e in self._search(
+ search_base=self.inst_search_base, search_filter=self.eligible_inst_filter,
+ attributes=['instID']
+ )
+ }
+
def get_managed_user_entries(self):
"""
Return a list containing all managed user entries as UserEntry instances.
@@ -104,6 +123,50 @@ class Configuration(ConfigurationDataclassMixin):
)
]
+ def get_managed_inst_entries(self):
+ """
+ Return a list containing all managed institution entries as GroupEntry instances.
+
+ Note that we return GroupEntry instances here since Lookup institutions become groups in
+ Google, and this simplifies the sync code by allowing us to handle institutions in the same
+ way as groups. The GroupEntry's groupID and groupName fields will be the institution's
+ instID and ou (name) respectively. Since Lookup institutions don't have descriptions, we
+ set the description field to the institution's name as well (in Google, the description
+ allows longer strings, and so will not truncate the name).
+
+ """
+ # This requires 2 LDAP queries. First find the managed institutions.
+ search_filter = (
+ self.managed_inst_filter
+ if self.managed_inst_filter is not None
+ else self.eligible_inst_filter
+ )
+ managed_insts = [
+ GroupEntry(
+ groupID=_extract(e, 'instID'), groupName=_extract(e, 'ou'),
+ description=_extract(e, 'ou'), uids=set(),
+ )
+ for e in self._search(
+ search_base=self.inst_search_base, search_filter=search_filter,
+ attributes=['instID', 'ou']
+ )
+ ]
+ managed_insts_by_instID = {g.groupID: g for g in managed_insts}
+
+ # Then get each eligible user's list of institutions and use that data to populate each
+ # institution's uid list.
+ eligible_users = self._search(
+ search_base=self.user_search_base, search_filter=self.eligible_user_filter,
+ attributes=['uid', 'instID']
+ )
+ for e in eligible_users:
+ uid = e['attributes']['uid'][0]
+ for instID in e['attributes']['instID']:
+ if instID in managed_insts_by_instID:
+ managed_insts_by_instID[instID].uids.add(uid)
+
+ return managed_insts
+
def _search(self, *, search_base, search_filter, attributes):
ldap_server = ldap3.Server(self.host)
with ldap3.Connection(ldap_server, auto_bind=True) as conn:
diff --git a/gsuitesync/sync.py b/gsuitesync/sync.py
index 66c5f2e..a7c5f61 100644
--- a/gsuitesync/sync.py
+++ b/gsuitesync/sync.py
@@ -110,37 +110,60 @@ def sync(configuration, *, read_only=True):
.with_subject(gapi_domain_config.admin_user)
)
- # Functions to translate the unique identifiers of users and groups in Lookup (uids and
- # groupIDs) to and from the unique identifiers used in Google (email addresses).
- #
- # For users: {uid} <-> {uid}@{domain}
- # For groups: {groupID} <-> {groupID}@{groups_domain}
- #
- # Additionally, valid uids (CRSids) match the regex [a-z][a-z0-9]{3,7}, and valid groupIDs
- # match the regex [0-9]{6,8}.
- user_email_regex = re.compile('^[a-z][a-z0-9]{3,7}@.*$')
- group_email_regex = re.compile('^[0-9]{6,8}@.*$')
-
+ # Secondary domain for Google groups that come from Lookup groups
groups_domain = (
gapi_domain_config.groups_domain
if gapi_domain_config.groups_domain is not None
else gapi_domain_config.name
)
+ # Secondary domain for Google groups that come from Lookup institutions
+ insts_domain = (
+ gapi_domain_config.insts_domain
+ if gapi_domain_config.insts_domain is not None
+ else gapi_domain_config.name
+ )
+
+ # Functions to translate the unique identifiers of users, groups and institutions in Lookup
+ # (uids, groupIDs and instIDs) to and from the unique identifiers used in Google (email
+ # addresses).
+ #
+ # For users: {uid} <-> {uid}@{domain}
+ # For groups: {groupID} <-> {groupID}@{groups_domain}
+ # For insts: {instID} <-> {instID.lower()}@{insts_domain} (local part must be lowercase)
+ #
+ # Additionally, valid uids (CRSids) match the regex [a-z][a-z0-9]{3,7}, valid groupIDs match
+ # the regex [0-9]{6,8} and valid instIDs match the regex [A-Z][A-Z0-9]+.
+ #
+ # Since Lookup institutions become groups in Google, we use common code to sync all Google
+ # groups, regardless of whether they were groups or institutions in Lookup. In all the code
+ # that follows, we use "gid" to refer to the unique identifier of the group or institution in
+ # Lookup (i.e., gid may be either a Lookup groupID or instID).
+ user_email_regex = re.compile('^[a-z][a-z0-9]{3,7}@.*$')
+ groupID_regex = re.compile('^[0-9]{6,8}$')
+ instID_regex = re.compile('^[A-Z][A-Z0-9]+$')
+
def uid_to_email(uid):
return f'{uid}@{gapi_domain_config.name}'
def email_to_uid(email):
return email.split('@')[0] if user_email_regex.match(email) else None
- def groupID_to_email(groupID):
- return f'{groupID}@{groups_domain}'
+ def gid_to_email(gid):
+ return (
+ f'{gid}@{groups_domain}' if groupID_regex.match(gid) else
+ f'{gid.lower()}@{insts_domain}' if instID_regex.match(gid) else None
+ )
- def email_to_groupID(email):
- return email.split('@')[0] if group_email_regex.match(email) else None
+ def email_to_gid(email):
+ gid = email.split('@')[0]
+ return (
+ gid if groupID_regex.match(gid) else
+ gid.upper() if instID_regex.match(gid.upper()) else None
+ )
# --------------------------------------------------------------------------------------------
- # Load current user and group data from Lookup.
+ # Load current user, group and institution data from Lookup.
# --------------------------------------------------------------------------------------------
# Get a set containing all CRSids. These are all the people who are eligible to be in our
@@ -157,6 +180,18 @@ def sync(configuration, *, read_only=True):
eligible_groupIDs = ldap_config.get_eligible_groupIDs()
LOG.info('Total LDAP group entries: %s', len(eligible_groupIDs))
+ # Get a set containing all instIDs. These are all the institutions that are eligible to be in
+ # our GSuite instance. If an institution is in GSuite and is *not* present in this list then
+ # the corresponding group is deleted.
+ LOG.info('Reading eligible institution entries from LDAP')
+ eligible_instIDs = ldap_config.get_eligible_instIDs()
+ LOG.info('Total LDAP institution entries: %s', len(eligible_instIDs))
+
+ # Add these sets together to form the set of all gids (the IDs of all eligible groups and
+ # institutions).
+ eligible_gids = eligible_groupIDs | eligible_instIDs
+ LOG.info('Total combined LDAP group and institution entries: %s', len(eligible_gids))
+
# Get a list of managed users. These are all the people who match the "managed_user_filter" in
# the LDAP settings.
LOG.info('Reading managed user entries from LDAP')
@@ -186,14 +221,41 @@ def sync(configuration, *, read_only=True):
LOG.info('Total managed group entries: %s', len(managed_group_groupIDs))
LOG.info('Total managed group members: %s', sum([len(g.uids) for g in managed_group_entries]))
+ # Get a list of managed institutions. These are all the institutions that match the
+ # "managed_inst_filter" in the LDAP settings.
+ LOG.info('Reading managed institution entries from LDAP')
+ managed_inst_entries = ldap_config.get_managed_inst_entries()
+
+ # Form a mapping from instID to managed institution.
+ managed_inst_entries_by_instID = {i.groupID: i for i in managed_inst_entries}
+
+ # Form a set of all *managed institution* instIDs
+ managed_inst_instIDs = set(managed_inst_entries_by_instID.keys())
+ LOG.info('Total managed institution entries: %s', len(managed_inst_instIDs))
+ LOG.info(
+ 'Total managed institution members: %s', sum([len(i.uids) for i in managed_inst_entries])
+ )
+
+ # Add the collections of managed institutions to the collections of managed groups.
+ managed_group_entries += managed_inst_entries
+ managed_group_entries_by_gid = {
+ **managed_group_entries_by_groupID, **managed_inst_entries_by_instID
+ }
+ managed_group_gids = managed_group_groupIDs | eligible_instIDs
+ LOG.info(
+ 'Total combined managed group and institution entries: %s', len(managed_group_gids)
+ )
+ LOG.info(
+ 'Total combined managed group and institution members: %s',
+ sum([len(g.uids) for g in managed_group_entries])
+ )
+
# Sanity check: the managed groups should be a subset of the eligible ones.
- if len(managed_group_groupIDs - eligible_groupIDs) != 0:
- raise RuntimeError(
- 'Sanity check failed: some managed groupIDs were not in the eligible set'
- )
+ if len(managed_group_gids - eligible_gids) != 0:
+ raise RuntimeError('Sanity check failed: some managed gids were not in the eligible set')
# --------------------------------------------------------------------------------------------
- # Load current user and group data from Google.
+ # Load current user, group and institution data from Google.
# --------------------------------------------------------------------------------------------
# Build the directory service using Google API discovery.
@@ -214,14 +276,28 @@ def sync(configuration, *, read_only=True):
retries=sync_config.http_retries, retry_delay=sync_config.http_retry_delay,
)
- # Retrieve information on all groups
+ # Function to fetch Google group information from the specified domain
+ def fetch_groups(domain):
+ fields = ['id', 'email', 'name', 'description']
+ return gapiutil.list_all(
+ directory_service.groups().list, items_key='groups', domain=domain,
+ fields='nextPageToken,groups(' + ','.join(fields) + ')',
+ retries=sync_config.http_retries, retry_delay=sync_config.http_retry_delay,
+ )
+
+ # Retrieve information on all Google groups that come from Lookup groups
LOG.info('Getting information on Google domain groups')
- fields = ['id', 'email', 'name', 'description']
- all_google_groups = gapiutil.list_all(
- directory_service.groups().list, items_key='groups', domain=groups_domain,
- fields='nextPageToken,groups(' + ','.join(fields) + ')',
- retries=sync_config.http_retries, retry_delay=sync_config.http_retry_delay,
- )
+ all_google_groups = [
+ g for g in fetch_groups(groups_domain)
+ if groupID_regex.match(g['email'].split('@')[0])
+ ]
+
+ # Append information on all Google groups that come from Lookup institutions
+ LOG.info('Getting information on Google domain institutions')
+ all_google_groups.extend([
+ g for g in fetch_groups(insts_domain)
+ if instID_regex.match(g['email'].split('@')[0].upper())
+ ])
# Strip any "to be ignored" users out of the results.
if sync_config.ignore_google_org_unit_path_regex is not None:
@@ -250,30 +326,31 @@ def sync(configuration, *, read_only=True):
all_google_users = [u for u in all_google_users if email_to_uid(u['primaryEmail'])]
# Strip out any groups whose email addresses don't match the pattern for groups created from
- # Lookup groupIDs, and which therefore should not be managed (deleted) by this script.
- all_google_groups = [g for g in all_google_groups if email_to_groupID(g['email'])]
+ # Lookup groupIDs or instIDs, and which therefore should not be managed (deleted) by this
+ # script.
+ all_google_groups = [g for g in all_google_groups if email_to_gid(g['email'])]
# Sanity check. There should be no admins in the returned results.
if any(u.get('isAdmin', False) for u in all_google_users):
raise RuntimeError('Sanity check failed: admin users in user list')
- # Form mappings from uid/groupID to Google user/group.
+ # Form mappings from uid/gid to Google user/group.
all_google_users_by_uid = {email_to_uid(u['primaryEmail']): u for u in all_google_users}
- all_google_groups_by_groupID = {email_to_groupID(g['email']): g for g in all_google_groups}
+ all_google_groups_by_gid = {email_to_gid(g['email']): g for g in all_google_groups}
- # Form sets of all Google-side uids and groupIDs. The all_google_uids set is all users
- # including the suspended ones and the suspended_google_uids set is only the suspended users.
- # Non suspended users are therefore all_google_uids - suspended_google_uids. Groups do not have
- # any concept of being suspended.
+ # Form sets of all Google-side uids and gids. The all_google_uids set is all users including
+ # the suspended ones and the suspended_google_uids set is only the suspended users. Non
+ # suspended users are therefore all_google_uids - suspended_google_uids. The all_google_gids
+ # set includes both groupIDs and instIDs. Groups in Google do not have any concept of being
+ # suspended.
all_google_uids = set(all_google_users_by_uid.keys())
- all_google_groupIDs = set(all_google_groups_by_groupID.keys())
+ all_google_gids = set(all_google_groups_by_gid.keys())
suspended_google_uids = {uid for uid, u in all_google_users_by_uid.items() if u['suspended']}
- # Sanity check. We should not have lost anything. (I.e. the uids and groupIDs should be
- # unique.)
+ # Sanity check. We should not have lost anything. (I.e. the uids and gids should be unique.)
if len(all_google_uids) != len(all_google_users):
raise RuntimeError('Sanity check failed: user list changed length')
- if len(all_google_groupIDs) != len(all_google_groups):
+ if len(all_google_gids) != len(all_google_groups):
raise RuntimeError('Sanity check failed: group list changed length')
# Retrieve all Google group settings.
@@ -285,17 +362,17 @@ def sync(configuration, *, read_only=True):
retries=sync_config.http_retries, retry_delay=sync_config.http_retry_delay,
)
- # Form a mapping from groupID to Google group settings.
- all_google_group_settings_by_groupID = {
- email_to_groupID(g['email']): g for g in all_google_group_settings
+ # Form a mapping from gid to Google group settings.
+ all_google_group_settings_by_gid = {
+ email_to_gid(g['email']): g for g in all_google_group_settings
}
# Santiy check. We should have settings for each managed group.
- if len(all_google_group_settings_by_groupID) != len(all_google_groups):
+ if len(all_google_group_settings_by_gid) != len(all_google_groups):
raise RuntimeError('Sanity check failed: group settings list does not match group list')
# Retrieve all Google group memberships. This is a mapping from internal Google group ids to
- # lists of member resources.
+ # lists of member resources, corresponding to both Lookup groups and institutions.
fields = ['id', 'email']
all_google_members = gapiutil.list_all_in_list(
directory_service, directory_service.members().list,
@@ -311,7 +388,7 @@ def sync(configuration, *, read_only=True):
# Log some stats.
LOG.info('Total Google users: %s', len(all_google_uids))
- LOG.info('Total Google groups: %s', len(all_google_groupIDs))
+ LOG.info('Total Google groups: %s', len(all_google_gids))
LOG.info(
'Suspended Google users: %s', sum(1 if u['suspended'] else 0 for u in all_google_users))
LOG.info(
@@ -377,12 +454,18 @@ def sync(configuration, *, read_only=True):
# if it needs updating/creating. If so, record a patch/insert for the group.
google_group_updates = {}
google_group_creations = {}
- for groupID, managed_group_entry in managed_group_entries_by_groupID.items():
+ for gid, managed_group_entry in managed_group_entries_by_gid.items():
# Form expected group resource fields. The 2 Google APIs we use here to update groups in
# Google each have different maximum lengths for group names and descriptions, and
# empirically the APIs don't function properly if either limit is exceeded, so we use the
# minimum of the 2 documented maximum field lengths (73 characters for names and 300
# characters for descriptions).
+ #
+ # Note that the source of each of these groups may be either a Lookup group or a Lookup
+ # institution, which are handled the same here. Technically Lookup institutions do not have
+ # descriptions, but the code in ldap.py sets the description from the name for Lookup
+ # institutions, which is useful since some institution names do not fit in the Google name
+ # field.
expected_google_group = {
'name': _trim_text(
managed_group_entry.groupName, maxlen=73, suffix=sync_config.group_name_suffix
@@ -391,7 +474,7 @@ def sync(configuration, *, read_only=True):
}
# Find existing Google group (if any).
- existing_google_group = all_google_groups_by_groupID.get(groupID)
+ existing_google_group = all_google_groups_by_gid.get(gid)
if existing_google_group is not None:
# See if we need to change the existing group
@@ -405,11 +488,11 @@ def sync(configuration, *, read_only=True):
# Only record non-empty patches.
if len(patch) > 0:
- google_group_updates[groupID] = patch
+ google_group_updates[gid] = patch
else:
# No existing Google group, so create one.
- google_group_creations[groupID] = {
- 'email': groupID_to_email(groupID),
+ google_group_creations[gid] = {
+ 'email': gid_to_email(gid),
**expected_google_group
}
@@ -417,17 +500,17 @@ def sync(configuration, *, read_only=True):
uids_to_update = set(google_user_updates.keys())
LOG.info('Number of existing users to update: %s', len(uids_to_update))
- # Form a set of all the groupIDs which need patching.
- groupIDs_to_update = set(google_group_updates.keys())
- LOG.info('Number of existing groups to update: %s', len(groupIDs_to_update))
+ # Form a set of all the gids which need patching.
+ gids_to_update = set(google_group_updates.keys())
+ LOG.info('Number of existing groups to update: %s', len(gids_to_update))
# Form a set of all the uids which need adding.
uids_to_add = set(google_user_creations.keys())
LOG.info('Number of users to add: %s', len(uids_to_add))
- # Form a set of all the groupIDs which need adding.
- groupIDs_to_add = set(google_group_creations.keys())
- LOG.info('Number of groups to add: %s', len(groupIDs_to_add))
+ # Form a set of all the gids which need adding.
+ gids_to_add = set(google_group_creations.keys())
+ LOG.info('Number of groups to add: %s', len(gids_to_add))
# Form a set of all uids which need reactivating. We reactive users who are in the managed user
# list *and* the suspended user list.
@@ -439,17 +522,17 @@ def sync(configuration, *, read_only=True):
uids_to_suspend = (all_google_uids - suspended_google_uids) - eligible_uids
LOG.info('Number of users to suspend: %s', len(uids_to_suspend))
- # Form a set of all groupIDs which need deleting.
- groupIDs_to_delete = all_google_groupIDs - eligible_groupIDs
- LOG.info('Number of groups to delete: %s', len(groupIDs_to_delete))
+ # Form a set of all gids which need deleting.
+ gids_to_delete = all_google_gids - eligible_gids
+ LOG.info('Number of groups to delete: %s', len(gids_to_delete))
# For each managed group, determine which members to insert or delete. These are lists of
- # (groupID, uid) tuples.
+ # (gid, uid) tuples.
members_to_insert = []
members_to_delete = []
- for groupID, managed_group_entry in managed_group_entries_by_groupID.items():
+ for gid, managed_group_entry in managed_group_entries_by_gid.items():
# Find the existing Google group members.
- existing_google_group = all_google_groups_by_groupID.get(groupID)
+ existing_google_group = all_google_groups_by_gid.get(gid)
if existing_google_group:
existing_members = all_google_members[existing_google_group['id']]
existing_member_uids = set([email_to_uid(m['email']) for m in existing_members])
@@ -461,7 +544,7 @@ def sync(configuration, *, read_only=True):
insert_uids = (
(managed_group_entry.uids - existing_member_uids).intersection(managed_user_uids)
)
- members_to_insert.extend([(groupID, uid) for uid in insert_uids])
+ members_to_insert.extend([(gid, uid) for uid in insert_uids])
# Members to delete. This is restricted to the eligible user set, so that we don't bother
# to delete a member resource when the user is suspended (and so we won't need to re-add
@@ -469,7 +552,7 @@ def sync(configuration, *, read_only=True):
delete_uids = (
(existing_member_uids - managed_group_entry.uids).intersection(eligible_uids)
)
- members_to_delete.extend([(groupID, uid) for uid in delete_uids])
+ members_to_delete.extend([(gid, uid) for uid in delete_uids])
LOG.info('Number of group members to insert: %s', len(members_to_insert))
LOG.info('Number of group members to delete: %s', len(members_to_delete))
@@ -487,9 +570,9 @@ def sync(configuration, *, read_only=True):
LOG.info('Configuration will modify %.2f%% of users', user_change_percentage)
group_change_percentage = 100. * (
- len(groupIDs_to_add | groupIDs_to_update | groupIDs_to_delete)
+ len(gids_to_add | gids_to_update | gids_to_delete)
/
- max(1, len(all_google_groupIDs))
+ max(1, len(all_google_gids))
)
LOG.info('Configuration will modify %.2f%% of groups', group_change_percentage)
@@ -533,22 +616,22 @@ def sync(configuration, *, read_only=True):
uids_to_add = capped_uids_to_add
LOG.info('Capped number of new users to %s', len(uids_to_add))
if (limits_config.max_new_groups is not None and
- len(groupIDs_to_add) > limits_config.max_new_groups):
+ len(gids_to_add) > limits_config.max_new_groups):
# Ensure that we do not attempt to insert a group member for any of the groups not added
# as a result of this cap, since these groups won't exist in Google
- capped_groupIDs_to_add = _limit(groupIDs_to_add, limits_config.max_new_groups)
- groupIDs_not_added = groupIDs_to_add - capped_groupIDs_to_add
- members_to_insert = [(g, u) for g, u in members_to_insert if g not in groupIDs_not_added]
- groupIDs_to_add = capped_groupIDs_to_add
- LOG.info('Capped number of new groups to %s', len(groupIDs_to_add))
+ capped_gids_to_add = _limit(gids_to_add, limits_config.max_new_groups)
+ gids_not_added = gids_to_add - capped_gids_to_add
+ members_to_insert = [(g, u) for g, u in members_to_insert if g not in gids_not_added]
+ gids_to_add = capped_gids_to_add
+ LOG.info('Capped number of new groups to %s', len(gids_to_add))
if (limits_config.max_suspended_users is not None and
len(uids_to_suspend) > limits_config.max_suspended_users):
uids_to_suspend = _limit(uids_to_suspend, limits_config.max_suspended_users)
LOG.info('Capped number of users to suspend to %s', len(uids_to_suspend))
if (limits_config.max_deleted_groups is not None and
- len(groupIDs_to_delete) > limits_config.max_deleted_groups):
- groupIDs_to_delete = _limit(groupIDs_to_delete, limits_config.max_deleted_groups)
- LOG.info('Capped number of groups to delete to %s', len(groupIDs_to_delete))
+ len(gids_to_delete) > limits_config.max_deleted_groups):
+ gids_to_delete = _limit(gids_to_delete, limits_config.max_deleted_groups)
+ LOG.info('Capped number of groups to delete to %s', len(gids_to_delete))
if (limits_config.max_reactivated_users is not None and
len(uids_to_reactivate) > limits_config.max_reactivated_users):
uids_to_reactivate = _limit(uids_to_reactivate, limits_config.max_reactivated_users)
@@ -558,9 +641,9 @@ def sync(configuration, *, read_only=True):
uids_to_update = _limit(uids_to_update, limits_config.max_updated_users)
LOG.info('Capped number of users to update to %s', len(uids_to_update))
if (limits_config.max_updated_groups is not None and
- len(groupIDs_to_update) > limits_config.max_updated_groups):
- groupIDs_to_update = _limit(groupIDs_to_update, limits_config.max_updated_groups)
- LOG.info('Capped number of groups to update to %s', len(groupIDs_to_update))
+ len(gids_to_update) > limits_config.max_updated_groups):
+ gids_to_update = _limit(gids_to_update, limits_config.max_updated_groups)
+ LOG.info('Capped number of groups to update to %s', len(gids_to_update))
if (limits_config.max_inserted_members is not None and
len(members_to_insert) > limits_config.max_inserted_members):
members_to_insert = members_to_insert[0:limits_config.max_inserted_members]
@@ -612,55 +695,55 @@ def sync(configuration, *, read_only=True):
# service to perform the actions required to update groups
def group_api_requests():
# Update existing groups
- group_updates = {groupID: google_group_updates[groupID] for groupID in groupIDs_to_update}
- for groupID, update in group_updates.items():
- google_id = all_google_groups_by_groupID[groupID]['id']
- LOG.info('Update group "%s": "%r"', groupID, update)
+ group_updates = {gid: google_group_updates[gid] for gid in gids_to_update}
+ for gid, update in group_updates.items():
+ google_id = all_google_groups_by_gid[gid]['id']
+ LOG.info('Update group "%s": "%r"', gid, update)
yield directory_service.groups().patch(groupKey=google_id, body=update)
# Delete cancelled groups
- for groupID in groupIDs_to_delete:
- google_id = all_google_groups_by_groupID[groupID]['id']
- LOG.info('Deleting group: "%s"', groupID)
+ for gid in gids_to_delete:
+ google_id = all_google_groups_by_gid[gid]['id']
+ LOG.info('Deleting group: "%s"', gid)
yield directory_service.groups().delete(groupKey=google_id)
# Create new groups
- for groupID in groupIDs_to_add:
- new_group = google_group_creations[groupID]
- LOG.info('Adding group "%s": %s', groupID, new_group)
+ for gid in gids_to_add:
+ new_group = google_group_creations[gid]
+ LOG.info('Adding group "%s": %s', gid, new_group)
yield directory_service.groups().insert(body=new_group)
# A generator which will generate patch() calls to the groupssettings service to set or
# update the required group settings.
def group_settings_api_requests():
# Apply all settings to new groups.
- for groupID in groupIDs_to_add:
- email = groupID_to_email(groupID)
+ for gid in gids_to_add:
+ email = gid_to_email(gid)
settings = sync_config.group_settings
- LOG.info('Updating settings for new group "%s": %s', groupID, settings)
+ LOG.info('Updating settings for new group "%s": %s', gid, settings)
yield groupssettings_service.groups().patch(groupUniqueId=email, body=settings)
# Apply any settings that differ to pre-existing groups.
- for groupID, settings in all_google_group_settings_by_groupID.items():
+ for gid, settings in all_google_group_settings_by_gid.items():
patch = {k: v for k, v in sync_config.group_settings.items() if settings.get(k) != v}
if patch:
- email = groupID_to_email(groupID)
- LOG.info('Updating settings for existing group "%s": %s', groupID, patch)
+ email = gid_to_email(gid)
+ LOG.info('Updating settings for existing group "%s": %s', gid, patch)
yield groupssettings_service.groups().patch(groupUniqueId=email, body=patch)
# A generator which will generate insert() and delete() calls to the directory service to
# perform the actions required to update group members
def member_api_requests():
# Insert new members
- for groupID, uid in members_to_insert:
- group_key = groupID_to_email(groupID)
+ for gid, uid in members_to_insert:
+ group_key = gid_to_email(gid)
user_key = uid_to_email(uid)
LOG.info('Adding user "%s" to group "%s"', user_key, group_key)
yield directory_service.members().insert(groupKey=group_key, body={'email': user_key})
# Delete removed members
- for groupID, uid in members_to_delete:
- group_key = groupID_to_email(groupID)
+ for gid, uid in members_to_delete:
+ group_key = gid_to_email(gid)
user_key = uid_to_email(uid)
LOG.info('Removing user "%s" from group "%s"', user_key, group_key)
yield directory_service.members().delete(groupKey=group_key, memberKey=user_key)
--
GitLab