FAQ | This is a LIVE service | Changelog

Commit 72d39e65 authored by Monty Dawson's avatar Monty Dawson
Browse files

Merge branch 'licensing' into 'master'

Add/Remove users Google Workspace licences

Closes #33

See merge request !24
parents 237b7261 aebaa360
Pipeline #169044 passed with stages
in 3 minutes and 15 seconds
......@@ -85,6 +85,10 @@ This tool requires the following OAuth2 scopes to actually perform changes:
* ``https://www.googleapis.com/auth/admin.directory.group``
* ``https://www.googleapis.com/auth/admin.directory.group.member``
Additionally, the following scope is required to view and manage licensing:
* ``https://www.googleapis.com/auth/apps.licensing``
See the section on preparing a service account for information on how to grant a
service account those scopes on your domain.
......@@ -99,6 +103,7 @@ to allow service accounts direct access to the API without needing domain-wide d
custom role with the following Admin **API** privileges (not Console privileges):
* Users - Create, Read, Update and Update Custom Attributes
* Groups - All privileges
* Licence Management - All privileges
4. Add the service account to the role using the "Assign service accounts" option
when viewing the custom role's admins
......
......@@ -65,6 +65,16 @@ limits:
# have no limit. Default: null.
abort_user_change_percentage: 2 # percent
# Refuse to perform sync if we are to "touch" more than this percentage of
# licence assignments. The percentage of licence assignments "touched" is
# calculated as
#
# (added licences + removed licences) / max(1, total licences)
#
# As such this calculated percentage can be greater than 100. Set to null to
# have no limit. Default: null.
abort_licence_change_percentage: 2 # percent
# Refuse to perform sync if we are to "touch" more than this percentage of
# groups. The percentage of groups "touched" is calculated as
#
......@@ -122,6 +132,14 @@ limits:
# number. Set to null to have no limit. Default: null
max_updated_users: 100
# Limit the number of licence assignments to create per run. This is an absolute number.
# Set to None to have no limit.
max_add_licence: 100
# Limit the number of licence assignments to delete per run. This is an absolute number.
# Set to None to have no limit.
max_remove_licence: 100
# Limit the number of group metadata changes per run. This is an absolute
# number. Set to null to have no limit. Default: null
max_updated_groups: 100
......@@ -230,3 +248,14 @@ google_domain:
# Secondary domain or domain alias for institutions. If null, the value of
# "name" is used. Default: null
insts_domain: null
# Details about Licensing assignment
licensing:
# Customer ID
customer_id: C01234578
# Product ID:
product_id: GSuite-Product
# Dict of SKU IDs with their maximum size, available for users to be assigned to
licensed_skus:
"SKU1": 10000
"SKU2": 3000
"""
Google Subscription licensing.
"""
import dataclasses
import typing
from .mixin import ConfigurationDataclassMixin
@dataclasses.dataclass
class Configuration(ConfigurationDataclassMixin):
"""
Configuration for applying subscription licensing to users in Google.
IDs and SKUs available from:
https://developers.google.com/admin-sdk/licensing/v1/how-tos/products
"""
# Customer ID for your Google Workspace domain. Licensing API cannot use `domain` like others.
customer_id: str
# Product ID of the Google Workspace product (e.g. 101031 for Google Workspace for Education)
# Use None if no subscriptions are allocated to the Google Workspace billing
product_id: typing.Optional[str] = None
# Dict of SKU IDs with their maximum size to assign licensed users to
# e.g.
# "1010310008": 30000 for Google Workspace for Education Plus
# "1010310009": 2000 for Google Workspace for Education Plus (Staff)
licensed_skus: typing.Dict[str, int] = dataclasses.field(default_factory=dict)
......@@ -27,6 +27,15 @@ class Configuration(ConfigurationDataclassMixin):
# calculated percentage can be greater than 100. Set to null to have no limit. Default: null.
abort_user_change_percentage: typing.Union[None, numbers.Real] = None
# Refuse to perform sync if we are to "touch" more than this percentage of licence
# assignments. The percentage of licence assignments "touched" is calculated as
#
# (added licences + removed licences) / max(1, total licences)
#
# As such this calculated percentage can be greater than 100. Set to null to have no limit.
# Default: null.
abort_licence_change_percentage: typing.Union[None, numbers.Real] = None
# Refuse to perform sync if we are to "touch" more than this percentage of groups. The
# percentage of groups "touched" is calculated as
#
......@@ -81,6 +90,14 @@ class Configuration(ConfigurationDataclassMixin):
# have no limit.
max_updated_users: typing.Union[None, numbers.Real] = None
# Limit the number of licence assignments to create per run. This is an absolute number. Set
# to None to have no limit.
max_add_licence: typing.Union[None, numbers.Real] = None
# Limit the number of licence assignments to delete per run. This is an absolute number. Set
# to None to have no limit.
max_remove_licence: typing.Union[None, numbers.Real] = None
# Limit the number of group metadata changes per run. This is an absolute number. Set to None
# to have no limit.
max_updated_groups: typing.Union[None, numbers.Real] = None
......
......@@ -5,7 +5,7 @@ import yaml
from .exceptions import ConfigurationNotFound
# Configuration declarations
from . import gapiauth, gapidomain, ldap, limits, sync
from . import gapiauth, gapidomain, ldap, limits, sync, licensing
LOG = logging.getLogger(__name__)
......@@ -56,4 +56,5 @@ def parse_configuration(configuration):
'limits': limits.Configuration.from_dict(configuration.get('limits', {})),
'gapi_auth': gapiauth.Configuration.from_dict(
configuration.get('google_api', {}).get('auth', {})),
'licensing': licensing.Configuration.from_dict(configuration.get('licensing', {})),
}
......@@ -10,12 +10,23 @@ from time import sleep
LOG = logging.getLogger(__name__)
def list_all(list_cb, *, page_size=500, retries=2, retry_delay=5, items_key='items', **kwargs):
class EmptyHttpResponse:
status = 400
reason = 'Empty Response'
class EmptyHttpResponseError(RuntimeError):
resp = EmptyHttpResponse()
content = b''
def list_all(list_cb, *, page_size=500, retries=2, retry_delay=5, items_key='items',
allow_empty=False, **kwargs):
"""
Simple wrapper for Google Client SDK list()-style callables. Repeatedly fetches pages of
results merging all the responses together. Returns the merged "items" arrays from the
responses. The key used to get the "items" array from the response may be overridden via the
items_key argument.
items_key argument. allow_empty determines if resulting resource list can be empty
"""
# Loop while we wait for nextPageToken to be "none"
......@@ -24,9 +35,11 @@ def list_all(list_cb, *, page_size=500, retries=2, retry_delay=5, items_key='ite
while True:
try:
list_response = list_cb(pageToken=page_token, maxResults=page_size, **kwargs).execute()
if not list_response:
raise HttpError({'status': 400, 'reason': 'Empty Response'}, b'')
except HttpError as err:
# Error if we have an empty response, except if we are allowing empty results and we
# don't have any results yet
if not list_response and (not allow_empty or resources):
raise EmptyHttpResponseError()
except (HttpError, EmptyHttpResponseError) as err:
if (err.resp.status >= 400 and retries > 0):
retries -= 1
LOG.warn('Error response: %s %s - retrying', err.resp.status, err.resp.reason)
......
......@@ -13,7 +13,7 @@ LOG = logging.getLogger(__name__)
class Comparator(ConfigurationStateConsumer):
required_config = ('gapi_domain', 'sync', 'limits')
required_config = ('gapi_domain', 'sync', 'limits', 'licensing')
def compare_users(self):
# For each user which exists in Google or the managed user set which is eligible,
......@@ -127,6 +127,55 @@ class Comparator(ConfigurationStateConsumer):
'uids_to_restore': uids_to_restore,
})
def compare_licensing(self):
# Compare Lookup and Googles licensed uids
if self.licensing_config.product_id is None:
LOG.info('Skipping licensing assignment comparison as no Product Id set')
return
# Determining what to do depending on set membership:
#
# eligible_uids | managed_uids | licensed_uids | google_licensed_uids | action
# ----------------------------------------------------------------------------
# yes | yes | yes | yes | nothing
# yes | yes | yes | no | add
# yes | yes | no | yes | remove [1]
# yes | yes | no | no | nothing
# yes | no | no | yes | nothing [2]
# yes | no | no | no | nothing
# no | no | no | yes | remove [3]
# no | no | no | no | nothing
#
# [1] - No longer a staff/student in lookup so remove google licence
# [2] - Not being managed so leave licence alone
# [3] - Not even eligible for account (now cancelled in lookup, already or will be
# suspended/deleted in google) so remove google licence
# Those licensed in Google but not in Lookup need removing but ignore those eligible
# but not managed
licences_to_remove = (
(self.state.google_licensed_uids - self.state.licensed_uids)
-
(self.state.eligible_uids - self.state.managed_user_uids)
)
LOG.info('Number of licence assignments to remove: %s', len(licences_to_remove))
# Those licensed in Lookup but not in Google need adding
licences_to_add = self.state.licensed_uids - self.state.google_licensed_uids
LOG.info('Number of licence assignments to add: %s', len(licences_to_add))
# Work out how many licences we will have available after updating
post_avail = (
sum(self.state.google_available_by_sku.values())
+ len(licences_to_remove)
- len(licences_to_add)
)
LOG.info('Number of licences available after update: %s', post_avail)
self.state.update({
'licences_to_remove': licences_to_remove,
'licences_to_add': licences_to_add,
})
def compare_groups(self):
# For each group which exists in Google or the managed group set which is eligible,
# determine if it needs updating/creating. If so, record a patch/insert for the group.
......@@ -270,6 +319,17 @@ class Comparator(ConfigurationStateConsumer):
)
LOG.info('Configuration will modify %.2f%% of users', user_change_percentage)
if self.licensing_config.product_id is not None:
# Calculate percentage change in licence assignments.
licence_change_percentage = 100. * (
len(self.state.licences_to_add) + len(self.state.licences_to_remove)
/
max(1, len(self.state.google_licensed_uids))
)
LOG.info(
'Configuration will modify %.2f%% of licence assignments',
licence_change_percentage)
if not just_users:
group_change_percentage = 100. * (
len(self.state.gids_to_add | self.state.gids_to_update | self.state.gids_to_delete)
......@@ -294,6 +354,18 @@ class Comparator(ConfigurationStateConsumer):
)
raise RuntimeError('Aborting due to large user change percentage')
if self.licensing_config.product_id is not None:
if (
self.limits_config.abort_licence_change_percentage is not None and
licence_change_percentage > self.limits_config.abort_licence_change_percentage
):
LOG.error(
'Modification of %.2f%% of licences is greater than limit of %.2f%%. '
'Aborting.',
licence_change_percentage, self.limits_config.abort_licence_change_percentage
)
raise RuntimeError('Aborting due to large licence assignment change percentage')
if not just_users:
if (self.limits_config.abort_group_change_percentage is not None and
group_change_percentage > self.limits_config.abort_group_change_percentage):
......@@ -361,6 +433,27 @@ class Comparator(ConfigurationStateConsumer):
)
LOG.info('Capped number of users to update to %s', len(self.state.uids_to_update))
# Licences
if self.licensing_config.product_id is not None:
if (self.limits_config.max_add_licence is not None and
len(self.state.licences_to_add) > self.limits_config.max_add_licence):
self.state.licences_to_add = _limit(
self.state.licences_to_add, self.limits_config.max_add_licence
)
LOG.info(
'Capped number of licences to assign to %s',
len(self.state.licences_to_add)
)
if (self.limits_config.max_remove_licence is not None and
len(self.state.licences_to_remove) > self.limits_config.max_remove_licence):
self.state.licences_to_remove = _limit(
self.state.licences_to_remove, self.limits_config.max_remove_licence
)
LOG.info(
'Capped number of licences to remove to %s',
len(self.state.licences_to_remove)
)
if not just_users:
if (self.limits_config.max_new_groups is not None and
len(self.state.gids_to_add) > self.limits_config.max_new_groups):
......
......@@ -20,19 +20,20 @@ READ_ONLY_SCOPES = [
'https://www.googleapis.com/auth/admin.directory.user.readonly',
'https://www.googleapis.com/auth/admin.directory.group.readonly',
'https://www.googleapis.com/auth/admin.directory.group.member.readonly',
'https://www.googleapis.com/auth/apps.groups.settings'
'https://www.googleapis.com/auth/apps.groups.settings',
'https://www.googleapis.com/auth/apps.licensing',
]
# Scopes *in addition to READ_ONLY_SCOPES* required to perform a full update.
WRITE_SCOPES = [
'https://www.googleapis.com/auth/admin.directory.user',
'https://www.googleapis.com/auth/admin.directory.group',
'https://www.googleapis.com/auth/admin.directory.group.member'
'https://www.googleapis.com/auth/admin.directory.group.member',
]
class GAPIRetriever(ConfigurationStateConsumer):
required_config = ('gapi_auth', 'gapi_domain', 'sync')
required_config = ('gapi_auth', 'gapi_domain', 'sync', 'licensing')
def connect(self, read_only=True):
# load credentials
......@@ -60,10 +61,16 @@ class GAPIRetriever(ConfigurationStateConsumer):
'groupssettings', 'v1', credentials=self.creds
)
# Also build the directory service for using the licensing API
licensing_service = discovery.build(
'licensing', 'v1', credentials=self.creds
)
# Return components needed for connection with Google API
self.state.update({
'directory_service': directory_service,
'groupssettings_service': groupssettings_service,
'licensing_service': licensing_service,
'groups_domain': groups_domain,
'insts_domain': insts_domain,
})
......@@ -158,6 +165,7 @@ class GAPIRetriever(ConfigurationStateConsumer):
domain=self.gapi_domain_config.name, showDeleted=show_deleted,
query='isAdmin=false', fields='nextPageToken,users(' + ','.join(fields) + ')',
retries=self.sync_config.http_retries, retry_delay=self.sync_config.http_retry_delay,
allow_empty=show_deleted,
)
# Strip any "to be ignored" users out of the results.
# Deleted users don't have an orgUnitPath.
......@@ -195,6 +203,57 @@ class GAPIRetriever(ConfigurationStateConsumer):
return all_google_users
def retrieve_licenses(self):
if self.licensing_config.product_id is None:
LOG.info('Skipping licensing assignment as no Product Id set')
return
# Retrieve all license assignments
LOG.info('Getting information on licence assignment for Google domain users')
fields = ['userId', 'skuId']
all_licence_assignments = gapiutil.list_all(
self.state.licensing_service.licenseAssignments().listForProduct,
customerId=self.licensing_config.customer_id,
productId=self.licensing_config.product_id,
fields='nextPageToken,items(' + ','.join(fields) + ')',
retries=self.sync_config.http_retries, retry_delay=self.sync_config.http_retry_delay,
)
LOG.info('Total licence assignments: %s', len(all_licence_assignments))
# Build a map of uids to license SKU. We are only interested in those in our domain
# and SKUs in our configuration
skus_by_uid = {
parts[1]: parts[0] for parts in
[[lic['skuId']] + lic['userId'].split('@', 1) for lic in all_licence_assignments]
if (
len(parts) == 3 and
parts[2] == self.gapi_domain_config.name and
parts[0] in self.licensing_config.licensed_skus.keys()
)
}
# Build the set of licensed uids
licensed_uids = set(skus_by_uid.keys())
LOG.info('Total licensed uids: %s', len(licensed_uids))
# Count available licences left for each SKU
available_by_sku = {
avail_sku: (total - len({
uid for uid, sku in skus_by_uid.items() if sku == avail_sku
}))
for avail_sku, total in self.licensing_config.licensed_skus.items()
}
for sku, available in available_by_sku.items():
LOG.info(f"Licences available in SKU '{sku}': {available}")
total_available = sum(available_by_sku.values())
LOG.info(f"Licences available in total: {total_available}")
self.state.update({
'google_licensed_uids': licensed_uids,
'google_skus_by_uid': skus_by_uid,
'google_available_by_sku': available_by_sku,
})
def retrieve_groups(self):
# Retrieve information on all Google groups that come from Lookup groups
LOG.info('Getting information on Google domain groups')
......
......@@ -11,7 +11,7 @@ from .base import ConfigurationStateConsumer
LOG = logging.getLogger(__name__)
# User and group information we need to populate the Google user directory.
UserEntry = collections.namedtuple('UserEntry', 'uid cn sn displayName givenName')
UserEntry = collections.namedtuple('UserEntry', 'uid cn sn displayName givenName licensed')
GroupEntry = collections.namedtuple('GroupEntry', 'groupID groupName description uids')
......@@ -48,10 +48,15 @@ class LDAPRetriever(ConfigurationStateConsumer):
'Sanity check failed: some managed uids were not in the eligible set'
)
# Form a set of licensed user uids (anyone with a non-empty misAffiliation)
licensed_uids = {u.uid for u in managed_user_entries if u.licensed}
LOG.info('Total licensed user entries: %s', len(licensed_uids))
self.state.update({
'eligible_uids': eligible_uids,
'managed_user_entries_by_uid': managed_user_entries_by_uid,
'managed_user_uids': managed_user_uids,
'licensed_uids': licensed_uids,
})
def retrieve_groups(self):
......@@ -189,11 +194,12 @@ class LDAPRetriever(ConfigurationStateConsumer):
return [
UserEntry(
uid=_extract(e, 'uid'), cn=_extract(e, 'cn'), sn=_extract(e, 'sn'),
displayName=_extract(e, 'displayName'), givenName=_extract(e, 'givenName')
displayName=_extract(e, 'displayName'), givenName=_extract(e, 'givenName'),
licensed=_extract_non_empty(e, 'misAffiliation')
)
for e in self._search(
search_base=self.ldap_config.user_search_base, search_filter=search_filter,
attributes=['uid', 'cn', 'sn', 'displayName', 'givenName']
attributes=['uid', 'cn', 'sn', 'displayName', 'givenName', 'misAffiliation']
)
]
......@@ -280,9 +286,23 @@ class LDAPRetriever(ConfigurationStateConsumer):
def _extract(entry, attr, *, default=''):
"""
Extract an attribute from an ldap entry, returning the attribute itself if single-valued
otherwise the first value of a multivalued attribute.
If the entry doesn't have the attribute then `default` is returned.
"""
vs = entry['attributes'].get(attr, [])
if len(vs) == 0:
return default
if isinstance(vs, str):
return vs
return vs[0]
def _extract_non_empty(entry, attr):
"""
Extract an attribute from an ldap entry, returning whether the attribute exists and is
not empty.
"""
return len(entry['attributes'].get(attr, [])) > 0
......@@ -38,6 +38,7 @@ def sync(configuration, *, read_only=True, group_settings=False, just_users=Fals
gapi = GAPIRetriever(configuration, state)
gapi.connect(read_only)
gapi.retrieve_users()
gapi.retrieve_licenses()
if not just_users:
gapi.retrieve_groups()
# Optionally get group settings too
......@@ -47,6 +48,7 @@ def sync(configuration, *, read_only=True, group_settings=False, just_users=Fals
# Compare users and optionally groups between Lookup and Google
comparator = Comparator(configuration, state)
comparator.compare_users()
comparator.compare_licensing()
if not just_users:
comparator.compare_groups()
# Optionally compare existing group settings too
......@@ -58,5 +60,6 @@ def sync(configuration, *, read_only=True, group_settings=False, just_users=Fals
# Update Google with necessary updates found doing comparison
updater = GAPIUpdater(configuration, state, read_only)
updater.update_users()
updater.update_licensing()
if not just_users:
updater.update_groups()
......@@ -17,6 +17,8 @@ class SyncState:
eligible_uids: set = field(default_factory=set)
managed_user_entries_by_uid: dict = field(default_factory=dict)
managed_user_uids: set = field(default_factory=set)
# licensing
licensed_uids: set = field(default_factory=set)
# group data
eligible_gids: set = field(default_factory=set)
managed_group_entries_by_gid: dict = field(default_factory=dict)
......@@ -26,6 +28,7 @@ class SyncState:
################
directory_service: Optional[discovery.Resource] = None
groupssettings_service: Optional[discovery.Resource] = None
licensing_service: Optional[discovery.Resource] = None
groups_domain: str = ''
insts_domain: str = ''
......@@ -40,6 +43,10 @@ class SyncState:
suspended_google_uids: set = field(default_factory=set)
never_logged_in_google_uids: set = field(default_factory=set)
all_deleted_google_users_by_uid: dict = field(default_factory=dict)
# licensing
google_licensed_uids: set = field(default_factory=set)
google_skus_by_uid: dict = field(default_factory=dict)
google_available_by_sku: dict = field(default_factory=dict)
# group data
all_google_groups: list = field(default_factory=list)
all_google_groups_by_gid: dict = field(default_factory=dict)
......@@ -62,6 +69,9 @@ class SyncState:
uids_to_suspend: set = field(default_factory=set)
uids_to_delete: set = field(default_factory=set)
uids_to_restore: set = field(default_factory=set)
# licensing
licences_to_remove: set = field(default_factory=set)
licences_to_add: set = field(default_factory=set)
# updates to groups
google_group_updates: dict = field(default_factory=dict)
google_group_creations: dict = field(default_factory=dict)
......
......@@ -14,7 +14,7 @@ LOG = logging.getLogger(__name__)
class GAPIUpdater(ConfigurationStateConsumer):
required_config = ('sync', 'gapi_domain')
required_config = ('sync', 'gapi_domain', 'licensing')
def __init__(self, configuration, state, read_only=True):
super(GAPIUpdater, self).__init__(configuration, state)
......@@ -26,6 +26,15 @@ class GAPIUpdater(ConfigurationStateConsumer):
self.user_api_requests(),
self.sync_config, self.read_only)
def update_licensing(self):
if self.licensing_config.product_id is None:
LOG.info('Skipping licensing assignment update as no Product Id set')
return
process_requests(
self.state.licensing_service,
self.licensing_api_requests(),
self.sync_config, self.read_only)
def update_groups(self):
process_requests(
self.state.directory_service,
......@@ -105,6 +114,44 @@ class GAPIUpdater(ConfigurationStateConsumer):
LOG.info('Adding user "%s": %s', uid, redacted_user)
yield self.state.directory_service.users().insert(body=new_user)
def licensing_api_requests(self):
"""
A generator which will generate insert() and delete() calls to the licensing
service to perform the actions required to update licensing
"""
# Tallying availability
available_by_sku = self.state.google_available_by_sku
def next_available_sku():
# Returns first sku with availability or None if all full
return next((sku for sku, avail in available_by_sku.items() if avail > 0), None)
# Remove licences
for uid in self.state.licences_to_remove:
userId = f'{uid}@{self.gapi_domain_config.name}'
skuId = self.state.google_skus_by_uid.get(uid)
LOG.info('Removing licence from user: "%s" (%s)', uid, skuId)
yield self.state.licensing_service.licenseAssignments().delete(
userId=userId, skuId=skuId, productId=self.licensing_config.product_id)
available_by_sku[skuId] += 1
# Add licences
for index, uid in enumerate(self.state.licences_to_add):
userId = f'{uid}@{self.gapi_domain_config.name}'
skuId = next_available_sku()
if skuId is None:
LOG.warning(
'No more available licences to allocate missing licenses for %d user(s)',
len(self.state.licences_to_add - index)
)
return
LOG.info('Adding licence to user: "%s" (%s)', uid, skuId)
yield self.state.licensing_service.licenseAssignments().insert(
skuId=skuId, productId=self.licensing_config.product_id,
body={'userId': userId})