From 286e0f0ddee462727861139450f561aeea169a90 Mon Sep 17 00:00:00 2001
From: av603 <av603@cam.ac.uk>
Date: Fri, 24 Mar 2023 16:13:10 +0000
Subject: [PATCH] delete users who are suspended and have a clean drive scan
result
---
CHANGELOG.md | 24 +-
configuration.yaml.example | 4 +-
gsuitesync/__init__.py | 56 ++--
gsuitesync/__main__.py | 2 +-
gsuitesync/config/api_gateway.py | 1 +
gsuitesync/config/exceptions.py | 3 +-
gsuitesync/config/gapiauth.py | 1 +
gsuitesync/config/gapidomain.py | 1 +
gsuitesync/config/licensing.py | 1 +
gsuitesync/config/limits.py | 1 +
gsuitesync/config/lookup.py | 1 +
gsuitesync/config/mixin.py | 7 +-
gsuitesync/config/sync.py | 32 +-
gsuitesync/config/utils.py | 40 +--
gsuitesync/gapiutil.py | 104 ++++---
gsuitesync/naming.py | 68 ++--
gsuitesync/sync/base.py | 7 +-
gsuitesync/sync/compare.py | 513 ++++++++++++++++++-------------
gsuitesync/sync/gapi.py | 389 +++++++++++++----------
gsuitesync/sync/lookup.py | 234 +++++++-------
gsuitesync/sync/main.py | 21 +-
gsuitesync/sync/state.py | 7 +-
gsuitesync/sync/update.py | 140 +++++----
gsuitesync/sync/utils.py | 40 ++-
requirements.txt | 4 +-
setup.py | 15 +-
tox.ini | 17 +-
27 files changed, 1012 insertions(+), 721 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ac3fd70..b00e9db 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,59 +5,67 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [0.14.0] - 2023-03-22
+
+### Added
+
+- Delete scanned users with no shared files.
+
## [0.13.0] - 2023-03-06
### Added
-- Custom Schema reading and writing to mark users' MyDrives to be scanned
+- Custom Schema reading and writing to mark users' MyDrives to be scanned
## [0.12.0] - 2022-12-08 - unreleased
-
## [0.11.1] - 2022-11-25
### Changed
-- Suspended members are removed from groups
+- Suspended members are removed from groups
## [0.11.0] - 2022-10-10
### Changed
-- Lookup users and groups retrieved via API Gateway instead of LDAP
+- Lookup users and groups retrieved via API Gateway instead of LDAP
## [0.10.0] - 2022-03-02
### Added
-- Support for licence management
+- Support for licence management
## [0.9.3] - 2020-11-10
### Changed
-- Updates to group settings conditional on --group-settings argument
+- Updates to group settings conditional on --group-settings argument
## [0.9.2] - 2020-11-10
### Added
+
- Group and Institution syncing
### Changed
-- Improved error handling
+- Improved error handling
## [0.9.1] - 2019-09-25
### Changed
+
- Improved display name usage
### Fixed
+
- Retry on error responses
- Set organisational unit when adding users
-
## [0.9.0] - 2019-04-30
### Added
+
- Initial implementation
diff --git a/configuration.yaml.example b/configuration.yaml.example
index aac65df..e7b97f3 100644
--- a/configuration.yaml.example
+++ b/configuration.yaml.example
@@ -222,8 +222,8 @@ lookup:
# Details about the Google Domain we're managing.
google_domain:
- # Name of the domain.
- name: 'example.com'
+ # Name of the domain. In this example, UCam test Google Workspace:
+ name: 'gdev.apps.cam.ac.uk'
# If using a service account with Domain-Wide Delegation, set to the username
# within the GSuite for the user which has administration rights.
diff --git a/gsuitesync/__init__.py b/gsuitesync/__init__.py
index 3e2b650..5fa3c5c 100644
--- a/gsuitesync/__init__.py
+++ b/gsuitesync/__init__.py
@@ -4,7 +4,8 @@ Synchronise users to GSuite
Usage:
gsuitesync (-h | --help)
gsuitesync [--configuration=FILE] [--quiet] [--group-settings] [--just-users]
- [--licensing] [--really-do-this] [--timeout=SECONDS] [--cache-dir=DIR]
+ [--licensing] [--delete-users] [--really-do-this] [--timeout=SECONDS]
+ [--cache-dir=DIR]
Options:
-h, --help Show a brief usage summary.
@@ -13,18 +14,22 @@ Options:
--configuration=FILE Specify configuration file to load.
- --group-settings Also update group settings on all groups
+ --group-settings Also update group settings on all groups.
- --just-users Just compare users not groups and institutions
+ --just-users Just compare users not groups and institutions.
- --licensing Also update user licence allocations
+ --licensing Also update user licence allocations.
+
+ --delete-users Delete suspended users.
--really-do-this Actually try to make the changes.
--timeout=SECONDS Integer timeout for socket when performing batch
operations. [default: 300]
- --cache-dir=DIR Directory to cache API responses, if provided
+ --cache-dir=DIR Directory to cache API responses, if provided.
+
+
"""
import logging
@@ -33,9 +38,7 @@ import sys
import docopt
-from . import config
-from . import sync
-
+from . import config, sync
LOG = logging.getLogger(os.path.basename(sys.argv[0]))
@@ -44,29 +47,34 @@ def main():
opts = docopt.docopt(__doc__)
# Configure logging
- logging.basicConfig(level=logging.WARN if opts['--quiet'] else logging.INFO)
+ logging.basicConfig(level=logging.WARN if opts["--quiet"] else logging.INFO)
# HACK: make the googleapiclient.discovery module less spammy in the logs
- logging.getLogger('googleapiclient.discovery').setLevel(logging.WARN)
- logging.getLogger('googleapiclient.discovery_cache').setLevel(logging.ERROR)
+ logging.getLogger("googleapiclient.discovery").setLevel(logging.WARN)
+ logging.getLogger("googleapiclient.discovery_cache").setLevel(logging.ERROR)
# Convert to integer, any raised exceptions will propagate upwards
- timeout_val = int(opts['--timeout'])
+ timeout_val = int(opts["--timeout"])
# Safety check - could write with out of date cached data
- if opts['--really-do-this'] and opts['--cache-dir']:
- LOG.error('Option --really-do-this with --cache-dir is blocked to avoid '
- 'writing outdated cached data.')
+ if opts["--really-do-this"] and opts["--cache-dir"]:
+ LOG.error(
+ "Option --really-do-this with --cache-dir is blocked to avoid "
+ "writing outdated cached data."
+ )
exit(1)
- LOG.info('Loading configuration')
- configuration = config.load_configuration(opts['--configuration'])
+ LOG.info("Loading configuration")
+ configuration = config.load_configuration(opts["--configuration"])
# Perform sync
- sync.sync(configuration,
- read_only=not opts['--really-do-this'],
- timeout=timeout_val,
- group_settings=opts['--group-settings'],
- just_users=opts['--just-users'],
- licensing=opts['--licensing'],
- cache_dir=opts['--cache-dir'])
+ sync.sync(
+ configuration,
+ read_only=not opts["--really-do-this"],
+ timeout=timeout_val,
+ group_settings=opts["--group-settings"],
+ just_users=opts["--just-users"],
+ licensing=opts["--licensing"],
+ delete_users=opts["--delete-users"],
+ cache_dir=opts["--cache-dir"],
+ )
diff --git a/gsuitesync/__main__.py b/gsuitesync/__main__.py
index c7c70d0..868d99e 100644
--- a/gsuitesync/__main__.py
+++ b/gsuitesync/__main__.py
@@ -1,4 +1,4 @@
from . import main
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
diff --git a/gsuitesync/config/api_gateway.py b/gsuitesync/config/api_gateway.py
index 79e682b..c643a2a 100644
--- a/gsuitesync/config/api_gateway.py
+++ b/gsuitesync/config/api_gateway.py
@@ -17,5 +17,6 @@ class Configuration(ConfigurationDataclassMixin):
Configuration of API Gateway access credentials.
"""
+
# Path to on-disk JSON credentials used when accessing APIs through API Gateway.
credentials: str
diff --git a/gsuitesync/config/exceptions.py b/gsuitesync/config/exceptions.py
index a6cb9b5..5c3ebe6 100644
--- a/gsuitesync/config/exceptions.py
+++ b/gsuitesync/config/exceptions.py
@@ -16,5 +16,6 @@ class ConfigurationNotFound(ConfigurationError):
A suitable configuration could not be located.
"""
+
def __init__(self):
- return super().__init__('Could not find any configuration file')
+ return super().__init__("Could not find any configuration file")
diff --git a/gsuitesync/config/gapiauth.py b/gsuitesync/config/gapiauth.py
index 11b62c2..0ea31a7 100644
--- a/gsuitesync/config/gapiauth.py
+++ b/gsuitesync/config/gapiauth.py
@@ -18,6 +18,7 @@ class Configuration(ConfigurationDataclassMixin):
Configuration of Google API access credentials.
"""
+
# Path to on-disk JSON credentials used when accessing the API.
credentials: str
diff --git a/gsuitesync/config/gapidomain.py b/gsuitesync/config/gapidomain.py
index 60389b0..d5f9c4a 100644
--- a/gsuitesync/config/gapidomain.py
+++ b/gsuitesync/config/gapidomain.py
@@ -14,6 +14,7 @@ class Configuration(ConfigurationDataclassMixin):
Configuration for accessing the Google Domain.
"""
+
# Name of the domain. (E.g. "example.com".)
name: str
diff --git a/gsuitesync/config/licensing.py b/gsuitesync/config/licensing.py
index 34e416f..0e52ef6 100644
--- a/gsuitesync/config/licensing.py
+++ b/gsuitesync/config/licensing.py
@@ -17,6 +17,7 @@ class Configuration(ConfigurationDataclassMixin):
https://developers.google.com/admin-sdk/licensing/v1/how-tos/products
"""
+
# Customer ID for your Google Workspace domain. Licensing API cannot use `domain` like others.
customer_id: str
diff --git a/gsuitesync/config/limits.py b/gsuitesync/config/limits.py
index e8e24d6..18b41ac 100644
--- a/gsuitesync/config/limits.py
+++ b/gsuitesync/config/limits.py
@@ -15,6 +15,7 @@ class Configuration(ConfigurationDataclassMixin):
Configuration for synchronisation limits.
"""
+
# The abort_... settings below are safety limits and will abort the run if the limits are
# violated. They are there to define the "sane limits" for an update.
diff --git a/gsuitesync/config/lookup.py b/gsuitesync/config/lookup.py
index b40f3f4..9d24d9d 100644
--- a/gsuitesync/config/lookup.py
+++ b/gsuitesync/config/lookup.py
@@ -15,6 +15,7 @@ class Configuration(ConfigurationDataclassMixin):
institutions in Lookup API.
"""
+
eligible_user_filter: str
eligible_group_filter: str
diff --git a/gsuitesync/config/mixin.py b/gsuitesync/config/mixin.py
index 0d54dad..8b452f3 100644
--- a/gsuitesync/config/mixin.py
+++ b/gsuitesync/config/mixin.py
@@ -15,16 +15,15 @@ class ConfigurationDataclassMixin:
"""
field_names = {field.name for field in dataclasses.fields(cls)}
required_field_names = {
- field.name for field in dataclasses.fields(cls)
- if field.default is dataclasses.MISSING
+ field.name for field in dataclasses.fields(cls) if field.default is dataclasses.MISSING
}
for key in dict_.keys():
if key not in field_names:
- raise ValueError(f'Unknown configuration key: {key}')
+ raise ValueError(f"Unknown configuration key: {key}")
for key in required_field_names:
if key not in dict_:
- raise ValueError(f'{key}: required field not set')
+ raise ValueError(f"{key}: required field not set")
return cls(**dict_)
diff --git a/gsuitesync/config/sync.py b/gsuitesync/config/sync.py
index f19771a..7f10b0a 100644
--- a/gsuitesync/config/sync.py
+++ b/gsuitesync/config/sync.py
@@ -19,28 +19,30 @@ class Configuration(ConfigurationDataclassMixin):
ignore_google_org_unit_path_regex: typing.Union[str, None] = None
# The organization unit path in which new accounts are placed
- new_user_org_unit_path: str = '/'
+ new_user_org_unit_path: str = "/"
# Suffix appended to the names of groups created in Google. The Google group name will be
# "{groupName}{group_name_suffix}", where {groupName} is the Lookup group name.
- group_name_suffix: str = ' from lookup.cam.ac.uk'
+ group_name_suffix: str = " from lookup.cam.ac.uk"
# Settings to be applied to groups in Google. These settings are applied to both new and
# existing groups imported from Lookup.
# See https://developers.google.com/admin-sdk/groups-settings/v1/reference/groups#json
- group_settings: dict = dataclasses.field(default_factory=lambda: {
- 'whoCanJoin': 'INVITED_CAN_JOIN',
- 'whoCanViewMembership': 'ALL_IN_DOMAIN_CAN_VIEW',
- 'whoCanViewGroup': 'ALL_MEMBERS_CAN_VIEW',
- 'whoCanPostMessage': 'ALL_IN_DOMAIN_CAN_POST',
- 'allowWebPosting': 'false',
- 'messageModerationLevel': 'MODERATE_ALL_MESSAGES',
- 'includeInGlobalAddressList': 'true',
- 'whoCanLeaveGroup': 'NONE_CAN_LEAVE',
- 'whoCanContactOwner': 'ALL_MANAGERS_CAN_CONTACT',
- 'whoCanModerateMembers': 'OWNERS_ONLY',
- 'whoCanDiscoverGroup': 'ALL_IN_DOMAIN_CAN_DISCOVER',
- })
+ group_settings: dict = dataclasses.field(
+ default_factory=lambda: {
+ "whoCanJoin": "INVITED_CAN_JOIN",
+ "whoCanViewMembership": "ALL_IN_DOMAIN_CAN_VIEW",
+ "whoCanViewGroup": "ALL_MEMBERS_CAN_VIEW",
+ "whoCanPostMessage": "ALL_IN_DOMAIN_CAN_POST",
+ "allowWebPosting": "false",
+ "messageModerationLevel": "MODERATE_ALL_MESSAGES",
+ "includeInGlobalAddressList": "true",
+ "whoCanLeaveGroup": "NONE_CAN_LEAVE",
+ "whoCanContactOwner": "ALL_MANAGERS_CAN_CONTACT",
+ "whoCanModerateMembers": "OWNERS_ONLY",
+ "whoCanDiscoverGroup": "ALL_IN_DOMAIN_CAN_DISCOVER",
+ }
+ )
# Inter-batch delay in seconds. This is useful to avoid hitting Google rate limits.
inter_batch_delay: numbers.Real = 5
diff --git a/gsuitesync/config/utils.py b/gsuitesync/config/utils.py
index 0dedd6b..aa96471 100644
--- a/gsuitesync/config/utils.py
+++ b/gsuitesync/config/utils.py
@@ -21,21 +21,23 @@ def load_configuration(location=None):
if location is not None:
paths = [location]
else:
- if 'GSUITESYNC_CONFIGURATION' in os.environ:
- paths = [os.environ['GSUITESYNC_CONFIGURATION']]
+ if "GSUITESYNC_CONFIGURATION" in os.environ:
+ paths = [os.environ["GSUITESYNC_CONFIGURATION"]]
else:
paths = []
- paths.extend([
- os.path.join(os.getcwd(), 'gsuitesync.yaml'),
- os.path.join(os.getcwd(), 'configuration.yaml'),
- os.path.expanduser('~/.gsuitesync/configuration.yaml'),
- '/etc/gsuitesync/configuration.yaml'
- ])
+ paths.extend(
+ [
+ os.path.join(os.getcwd(), "gsuitesync.yaml"),
+ os.path.join(os.getcwd(), "configuration.yaml"),
+ os.path.expanduser("~/.gsuitesync/configuration.yaml"),
+ "/etc/gsuitesync/configuration.yaml",
+ ]
+ )
valid_paths = [path for path in paths if os.path.isfile(path)]
if len(valid_paths) == 0:
- LOG.error('Could not find configuration file. Tried:')
+ LOG.error("Could not find configuration file. Tried:")
for path in paths:
LOG.error('"%s"', path)
raise ConfigurationNotFound()
@@ -51,13 +53,15 @@ def parse_configuration(configuration):
"""
return {
- 'api_gateway_auth': api_gateway.Configuration.from_dict(
- configuration.get('api_gateway', {}).get('auth', {})),
- 'sync': sync.Configuration.from_dict(configuration.get('sync', {})),
- 'gapi_domain': gapidomain.Configuration.from_dict(configuration.get('google_domain', {})),
- 'limits': limits.Configuration.from_dict(configuration.get('limits', {})),
- 'lookup': lookup.Configuration.from_dict(configuration.get('lookup', {})),
- 'gapi_auth': gapiauth.Configuration.from_dict(
- configuration.get('google_api', {}).get('auth', {})),
- 'licensing': licensing.Configuration.from_dict(configuration.get('licensing', {})),
+ "api_gateway_auth": api_gateway.Configuration.from_dict(
+ configuration.get("api_gateway", {}).get("auth", {})
+ ),
+ "sync": sync.Configuration.from_dict(configuration.get("sync", {})),
+ "gapi_domain": gapidomain.Configuration.from_dict(configuration.get("google_domain", {})),
+ "limits": limits.Configuration.from_dict(configuration.get("limits", {})),
+ "lookup": lookup.Configuration.from_dict(configuration.get("lookup", {})),
+ "gapi_auth": gapiauth.Configuration.from_dict(
+ configuration.get("google_api", {}).get("auth", {})
+ ),
+ "licensing": licensing.Configuration.from_dict(configuration.get("licensing", {})),
}
diff --git a/gsuitesync/gapiutil.py b/gsuitesync/gapiutil.py
index 288ca65..2a4f8f0 100644
--- a/gsuitesync/gapiutil.py
+++ b/gsuitesync/gapiutil.py
@@ -12,16 +12,24 @@ LOG = logging.getLogger(__name__)
class EmptyHttpResponse:
status = 400
- reason = 'Empty Response'
+ reason = "Empty Response"
class EmptyHttpResponseError(RuntimeError):
resp = EmptyHttpResponse()
- content = b''
-
-
-def list_all(list_cb, *, page_size=500, retries=2, retry_delay=5, items_key='items',
- allow_empty=False, **kwargs):
+ content = b""
+
+
+def list_all(
+ list_cb,
+ *,
+ page_size=500,
+ retries=2,
+ retry_delay=5,
+ items_key="items",
+ allow_empty=False,
+ **kwargs,
+):
"""
Simple wrapper for Google Client SDK list()-style callables. Repeatedly fetches pages of
results merging all the responses together. Returns the merged "items" arrays from the
@@ -41,32 +49,44 @@ def list_all(list_cb, *, page_size=500, retries=2, retry_delay=5, items_key='ite
if not list_response and not (resources or allow_empty):
raise EmptyHttpResponseError()
except (HttpError, EmptyHttpResponseError) as err:
- if (err.resp.status >= 400 and retries > 0):
+ if err.resp.status >= 400 and retries > 0:
retries -= 1
- LOG.warn('Error response: %s %s - retrying', err.resp.status, err.resp.reason)
+ LOG.warn("Error response: %s %s - retrying", err.resp.status, err.resp.reason)
sleep(retry_delay)
continue
if retries == 0:
LOG.error(
- 'Error response: %s %s - retry count exceeded', err.resp.status,
- err.resp.reason
+ "Error response: %s %s - retry count exceeded",
+ err.resp.status,
+ err.resp.reason,
)
- LOG.error('Error content: %r', err.content)
+ LOG.error("Error content: %r", err.content)
raise
items = list_response.get(items_key, [])
resources.extend(items)
- LOG.info(f'Fetched page with {len(items)} items - total {len(resources)}')
+ LOG.info(f"Fetched page with {len(items)} items - total {len(resources)}")
# Get the token for the next page
- page_token = list_response.get('nextPageToken')
+ page_token = list_response.get("nextPageToken")
if page_token is None:
break
return resources
-def list_all_in_list(directory_service, list_cb, *, item_ids=[], id_key='key', batch_size=1000,
- page_size=500, retries=2, retry_delay=5, items_key='items', **kwargs):
+def list_all_in_list(
+ directory_service,
+ list_cb,
+ *,
+ item_ids=[],
+ id_key="key",
+ batch_size=1000,
+ page_size=500,
+ retries=2,
+ retry_delay=5,
+ items_key="items",
+ **kwargs,
+):
"""
Wrapper for Google Client SDK list()-style callables, operating on a list of items. Invokes
the "list_cb" Google API method for each item in the "item_ids" list, repeatedly fetching
@@ -85,7 +105,7 @@ def list_all_in_list(directory_service, list_cb, *, item_ids=[], id_key='key', b
# https://developers.google.com/admin-sdk/directory/v1/guides/batch
resources = {item_id: [] for item_id in item_ids}
for i in range(0, len(item_ids), batch_size):
- batch_item_ids = item_ids[i:i+batch_size]
+ batch_item_ids = item_ids[i : i + batch_size]
# New requests needed to get the next page of results for any item in this batch
new_requests = {}
@@ -97,7 +117,7 @@ def list_all_in_list(directory_service, list_cb, *, item_ids=[], id_key='key', b
resources[item_id].extend(response.get(items_key, []))
# Build a new request for the next page, if there is one
- page_token = response.get('nextPageToken')
+ page_token = response.get("nextPageToken")
if page_token:
request = list_cb(
pageToken=page_token, maxResults=page_size, **kwargs, **{id_key: item_id}
@@ -118,15 +138,15 @@ def list_all_in_list(directory_service, list_cb, *, item_ids=[], id_key='key', b
try:
batch.execute()
except HttpError as err:
- if (err.resp.status >= 400 and retries > 0):
+ if err.resp.status >= 400 and retries > 0:
retries -= 1
- LOG.warn(f'{err.resp.status}: error - retrying')
+ LOG.warn(f"{err.resp.status}: error - retrying")
sleep(retry_delay)
continue
if retries == 0:
- LOG.error(f'{err.resp.status}: error - retry count exceeded')
+ LOG.error(f"{err.resp.status}: error - retry count exceeded")
raise
- LOG.info(f'Batch processing of {actual_batch_size} items completed')
+ LOG.info(f"Batch processing of {actual_batch_size} items completed")
break
# Form a new batch for the next page of results for each item, if any
@@ -142,8 +162,17 @@ def list_all_in_list(directory_service, list_cb, *, item_ids=[], id_key='key', b
return resources
-def get_all_in_list(directory_service, get_cb, *, item_ids=[], id_key='key', batch_size=1000,
- retries=2, retry_delay=5, **kwargs):
+def get_all_in_list(
+ directory_service,
+ get_cb,
+ *,
+ item_ids=[],
+ id_key="key",
+ batch_size=1000,
+ retries=2,
+ retry_delay=5,
+ **kwargs,
+):
"""
Wrapper for Google Client SDK get()-style callables, operating on a list of items. Invokes the
"get_cb" Google API method for each item in the "item_ids" list, returning a list resources.
@@ -166,7 +195,7 @@ def get_all_in_list(directory_service, get_cb, *, item_ids=[], id_key='key', bat
resources.append(response)
for i in range(0, len(item_ids), batch_size):
- batch_item_ids = item_ids[i:i+batch_size]
+ batch_item_ids = item_ids[i : i + batch_size]
# Form the batch request
batch = directory_service.new_batch_http_request()
@@ -180,19 +209,20 @@ def get_all_in_list(directory_service, get_cb, *, item_ids=[], id_key='key', bat
try:
batch.execute()
except HttpError as err:
- if (err.resp.status >= 400 and retries > 0):
+ if err.resp.status >= 400 and retries > 0:
retries -= 1
- LOG.warn('Error response: %s %s - retrying', err.resp.status, err.resp.reason)
+ LOG.warn("Error response: %s %s - retrying", err.resp.status, err.resp.reason)
sleep(retry_delay)
continue
if retries == 0:
LOG.error(
- 'Error response: %s %s - retry count exceeded', err.resp.status,
- err.resp.reason
+ "Error response: %s %s - retry count exceeded",
+ err.resp.status,
+ err.resp.reason,
)
- LOG.error('Error content: %r', err.content)
+ LOG.error("Error content: %r", err.content)
raise
- LOG.info(f'Batch processing of {actual_batch_size} items completed')
+ LOG.info(f"Batch processing of {actual_batch_size} items completed")
break
return resources
@@ -214,30 +244,30 @@ def process_requests(service, requests, sync_config, read_only=True):
# Execute the batch request if not in read only mode. Otherwise log that we would
# have.
if not read_only:
- LOG.info('Issuing batch request to Google.')
+ LOG.info("Issuing batch request to Google.")
sleep(sync_config.inter_batch_delay)
retries = sync_config.http_retries
while True:
try:
batch.execute()
except HttpError as err:
- if (err.resp.status == 503 and retries > 0):
+ if err.resp.status == 503 and retries > 0:
retries -= 1
- LOG.warn('503: Service unavailable - retrying')
+ LOG.warn("503: Service unavailable - retrying")
sleep(sync_config.http_retry_delay)
continue
if retries == 0:
- LOG.error('503: Service unavailable - retry count exceeded')
+ LOG.error("503: Service unavailable - retry count exceeded")
raise
break
else:
- LOG.info('Not issuing batch request in read-only mode.')
+ LOG.info("Not issuing batch request in read-only mode.")
def _handle_batch_response(request_id, response, exception):
if exception is not None:
- LOG.error('Error performing request: %s', exception)
- LOG.error('Response: %r', response)
+ LOG.error("Error performing request: %s", exception)
+ LOG.error("Response: %r", response)
def _grouper(iterable, *, n):
diff --git a/gsuitesync/naming.py b/gsuitesync/naming.py
index a9f9740..86f60f8 100644
--- a/gsuitesync/naming.py
+++ b/gsuitesync/naming.py
@@ -7,7 +7,7 @@ import re
# The human-friendly names constructed by get_names().
-Names = collections.namedtuple('Names', 'given_name family_name')
+Names = collections.namedtuple("Names", "given_name family_name")
def get_names(*, uid, display_name=None, cn=None, sn=None, given_name=None):
@@ -128,19 +128,15 @@ def get_names(*, uid, display_name=None, cn=None, sn=None, given_name=None):
cn = cn.strip() if cn is not None and cn != uid else None
sn = sn.strip() if sn is not None and sn != uid else None
display_name = (
- display_name.strip()
- if display_name is not None and display_name != uid else None
- )
- given_name = (
- given_name.strip()
- if given_name is not None and given_name != uid else None
+ display_name.strip() if display_name is not None and display_name != uid else None
)
+ given_name = given_name.strip() if given_name is not None and given_name != uid else None
# If any of cn, sn, display_name or given_name are blank, proceed as it they're not set.
- cn = cn if cn != '' else None
- sn = sn if sn != '' else None
- display_name = display_name if display_name != '' else None
- given_name = given_name if given_name != '' else None
+ cn = cn if cn != "" else None
+ sn = sn if sn != "" else None
+ display_name = display_name if display_name != "" else None
+ given_name = given_name if given_name != "" else None
# Function to construct return value from family name and given name. Google names can't be
# longer than 60 characters so truncate them after cleaning.
@@ -149,9 +145,11 @@ def get_names(*, uid, display_name=None, cn=None, sn=None, given_name=None):
# If we have a display name and it doesn't match cn (or we have no cn) then use this as
# an indication that this is what the user wants displayed
- if (display_name is not None
- and (cn is None or cn != display_name)
- and display_name != jd_import_default(given_name, sn)):
+ if (
+ display_name is not None
+ and (cn is None or cn != display_name)
+ and display_name != jd_import_default(given_name, sn)
+ ):
gn, fn = _split_fullname(display_name, sn=sn, given_name=given_name)
if gn:
return _make_ret(family_name=fn, given_name=gn)
@@ -173,13 +171,13 @@ def get_names(*, uid, display_name=None, cn=None, sn=None, given_name=None):
return _make_ret(family_name=fn, given_name=gn)
# Support Wookey.
- if display_name is not None and ' ' not in display_name:
+ if display_name is not None and " " not in display_name:
return _make_ret(family_name=uid, given_name=display_name)
- if sn is not None and ' ' not in sn:
+ if sn is not None and " " not in sn:
return _make_ret(family_name=sn, given_name=uid)
- if given_name is not None and ' ' not in given_name:
+ if given_name is not None and " " not in given_name:
return _make_ret(family_name=uid, given_name=given_name)
- if cn is not None and ' ' not in cn:
+ if cn is not None and " " not in cn:
return _make_ret(family_name=uid, given_name=cn)
# Give up and return uid for both fields
@@ -233,39 +231,39 @@ def _split_fullname(fullname, sn=None, given_name=None):
"""
# If we have a sn and fullname ends with it, split out the sn.
if sn is not None and fullname.endswith(sn):
- gn = fullname[:-len(sn)].strip()
- if gn != '':
+ gn = fullname[: -len(sn)].strip()
+ if gn != "":
return (gn, sn)
# If we have a sn with hyphens and fullname ends with sn with hyphens replaced by
# spaces then split out this from fullname
- if sn is not None and '-' in sn:
- spaced_sn = sn.replace('-', ' ')
+ if sn is not None and "-" in sn:
+ spaced_sn = sn.replace("-", " ")
if fullname.endswith(spaced_sn):
- gn = fullname[:-len(spaced_sn)].strip()
- if gn != '':
+ gn = fullname[: -len(spaced_sn)].strip()
+ if gn != "":
return (gn, spaced_sn)
# If we have a given_name and fullname starts with it, split out the given_name
# then check for initials
if given_name is not None and fullname.startswith(given_name):
- gn = fullname[:len(given_name)].strip()
- fn = fullname[len(given_name):].strip()
+ gn = fullname[: len(given_name)].strip()
+ fn = fullname[len(given_name) :].strip()
first_initial = True
- while re.match(r'^[A-Z]\.', fn):
- gn += ' ' if first_initial else ''
+ while re.match(r"^[A-Z]\.", fn):
+ gn += " " if first_initial else ""
first_initial = False
gn += fn[:2]
fn = fn[2:].strip()
- if fn != '':
+ if fn != "":
return (gn, fn)
# split at last space and see if we have two parts
components = fullname.split()
if len(components) > 0:
fn = components[-1]
- gn = ' '.join(components[:-1])
- if gn != '' and fn != '':
+ gn = " ".join(components[:-1])
+ if gn != "" and fn != "":
return (gn, fn)
return (None, None)
@@ -297,11 +295,11 @@ def jd_import_default(gn, sn):
"""
if gn is None or sn is None:
- return ''
+ return ""
- initial = '-'.join([name[0].upper() for name in gn.split('-')]) + "."
+ initial = "-".join([name[0].upper() for name in gn.split("-")]) + "."
- return f'{initial} {sn}'
+ return f"{initial} {sn}"
def _clean(s):
@@ -326,7 +324,7 @@ def _clean(s):
'a b c'
"""
- return ''.join(c for c in s if c not in _CLEAN_BAD_CHARS)
+ return "".join(c for c in s if c not in _CLEAN_BAD_CHARS)
# Characters stripped by _clean. Present as a constant to avoid re-creating it.
diff --git a/gsuitesync/sync/base.py b/gsuitesync/sync/base.py
index b0607b1..adb6296 100644
--- a/gsuitesync/sync/base.py
+++ b/gsuitesync/sync/base.py
@@ -7,10 +7,11 @@ Base classes for retrievers, comparator and updater classes that consume configu
class ConfigurationStateConsumer:
required_config = None
- def __init__(self, configuration, state, read_only=True, cache_dir=None):
+ def __init__(self, configuration, state, read_only=True, delete_users=False, cache_dir=None):
# For convenience, create properties for required configuration
- for c in (self.required_config if self.required_config is not None else []):
- setattr(self, f'{c}_config', configuration.get(c, {}))
+ for c in self.required_config if self.required_config is not None else []:
+ setattr(self, f"{c}_config", configuration.get(c, {}))
self.state = state
self.read_only = read_only
+ self.delete_users = delete_users
self.cache_dir = cache_dir
diff --git a/gsuitesync/sync/compare.py b/gsuitesync/sync/compare.py
index eea551d..c5a1a0d 100644
--- a/gsuitesync/sync/compare.py
+++ b/gsuitesync/sync/compare.py
@@ -9,30 +9,36 @@ from .. import naming
from .base import ConfigurationStateConsumer
from .utils import uid_to_email, gid_to_email, email_to_uid
+from .gapi import MYDRIVE_SHARED_RESULT_NONE, MYDRIVE_SHARED_RESULT_REMOVED
+
LOG = logging.getLogger(__name__)
class Comparator(ConfigurationStateConsumer):
- required_config = ('gapi_domain', 'sync', 'limits', 'licensing')
+ required_config = ("gapi_domain", "sync", "limits", "licensing")
def compare_users(self):
# For each user which exists in Google or the managed user set which is eligible,
# determine if they need updating/creating. If so, record a patch/insert for the user.
- LOG.info('Calculating updates to users...')
+ LOG.info("Calculating updates to users...")
google_user_updates = {}
google_user_creations = {}
google_user_restores = {}
for uid, managed_user_entry in self.state.managed_user_entries_by_uid.items():
# Heuristically determine the given and family names.
names = naming.get_names(
- uid=uid, display_name=managed_user_entry.displayName, cn=managed_user_entry.cn,
- sn=managed_user_entry.sn, given_name=managed_user_entry.givenName)
+ uid=uid,
+ display_name=managed_user_entry.displayName,
+ cn=managed_user_entry.cn,
+ sn=managed_user_entry.sn,
+ given_name=managed_user_entry.givenName,
+ )
# Form expected user resource fields.
expected_google_user = {
- 'name': {
- 'givenName': names.given_name,
- 'familyName': names.family_name,
+ "name": {
+ "givenName": names.given_name,
+ "familyName": names.family_name,
},
}
@@ -46,21 +52,21 @@ class Comparator(ConfigurationStateConsumer):
# It is not possible to update/reactivate the user in the same pass as they are
# deleted as there is a delay in them becoming available. Undeleted users will
# therefore remain suspended until the next pass.
- google_user_restores[uid] = existing_deleted_user['id']
+ google_user_restores[uid] = existing_deleted_user["id"]
elif existing_google_user is not None:
# See if we need to change the existing user
# Unless anything needs changing, the patch is empty.
patch = {}
# Determine how to patch user's name.
- google_name = existing_google_user.get('name', {})
+ google_name = existing_google_user.get("name", {})
patch_name = {}
- if google_name.get('givenName') != expected_google_user['name']['givenName']:
- patch_name['givenName'] = names.given_name
- if google_name.get('familyName') != expected_google_user['name']['familyName']:
- patch_name['familyName'] = names.family_name
+ if google_name.get("givenName") != expected_google_user["name"]["givenName"]:
+ patch_name["givenName"] = names.given_name
+ if google_name.get("familyName") != expected_google_user["name"]["familyName"]:
+ patch_name["familyName"] = names.family_name
if len(patch_name) > 0:
- patch['name'] = patch_name
+ patch["name"] = patch_name
# Only record non-empty patches.
if len(patch) > 0:
@@ -71,45 +77,33 @@ class Comparator(ConfigurationStateConsumer):
# have the user log in with it. For password-only applications the user can make
# use of an application-specific password.
new_user = {
- 'primaryEmail': uid_to_email(uid, self.gapi_domain_config.name),
+ "primaryEmail": uid_to_email(uid, self.gapi_domain_config.name),
**expected_google_user,
}
google_user_creations[uid] = new_user
# Form a set of all the uids which need restoring.
uids_to_restore = set(google_user_restores.keys())
- LOG.info('Number of deleted users to restore: %s', len(uids_to_restore))
+ LOG.info("Number of deleted users to restore: %s", len(uids_to_restore))
# Form a set of all the uids which need patching.
uids_to_update = set(google_user_updates.keys())
- LOG.info('Number of existing users to update: %s', len(uids_to_update))
+ LOG.info("Number of existing users to update: %s", len(uids_to_update))
# Form a set of all the uids which need adding.
uids_to_add = set(google_user_creations.keys())
- LOG.info('Number of users to add: %s', len(uids_to_add))
+ LOG.info("Number of users to add: %s", len(uids_to_add))
# Form a set of all uids which need reactivating. We reactive users who are in the managed
# user list *and* the suspended user list.
uids_to_reactivate = self.state.suspended_google_uids & self.state.managed_user_uids
- LOG.info('Number of users to reactivate: %s', len(uids_to_reactivate))
+ LOG.info("Number of users to reactivate: %s", len(uids_to_reactivate))
# Form a set of all uids which should be suspended. This is all the unsuspended Google uids
# which do not appear in our eligible user list.
uids_to_suspend = (
- (self.state.all_google_uids - self.state.suspended_google_uids)
- - self.state.eligible_uids
- )
-
- # Form a set of all uids which should be deleted. This is all the already suspended Google
- # uids (except those to be reactivated) plus the uids to be suspended, who have never
- # logged in.
- # Also remove them from the uids to be suspended - no point suspending then deleting them
- uids_to_delete = (
- ((self.state.suspended_google_uids - uids_to_reactivate) | uids_to_suspend)
- & self.state.never_logged_in_google_uids
- )
- delete_instead = uids_to_suspend & uids_to_delete
- uids_to_suspend = uids_to_suspend - uids_to_delete
+ self.state.all_google_uids - self.state.suspended_google_uids
+ ) - self.state.eligible_uids
# Users that need their MyDrive scanned for shared files, are those suspended users who
# haven't logged in to Google for 6 months, have a Lookup cancelledDate over 6 months,
@@ -117,34 +111,86 @@ class Comparator(ConfigurationStateConsumer):
# reactivated.
uids_to_scan_mydrive = (
- self.state.never_logged_in_six_months_uids &
- self.state.cancelled_six_months_ago_uids &
- self.state.no_mydrive_shared_settings_uids
+ self.state.never_logged_in_six_months_uids
+ & self.state.cancelled_six_months_ago_uids
+ & self.state.no_mydrive_shared_settings_uids
) - uids_to_reactivate
- LOG.info('Number of users to suspend: %s', len(uids_to_suspend))
+ # Form a set of all uids who have never logged in and should be deleted. This is all the
+ # already suspended Google uids (except those to be reactivated) plus the uids to be
+ # suspended, who have never logged in.
+ # Also remove them from the uids to be suspended - no point suspending then deleting them
+
+ uids_to_delete_who_never_logged_in = set()
+ if self.delete_users:
+ uids_to_delete_who_never_logged_in = (
+ self.state.suspended_google_uids - uids_to_reactivate
+ ) & self.state.never_logged_in_google_uids
+
+ # Form a set of all uids who have already been suspended, have a mydrive scan result
+ # `permissions-none` and have been suspended in lookup for more than 24 months:
+ uids_to_delete_with_mydrive_shared_result_permission_none = set()
+ if self.delete_users:
+ uids_to_delete_with_mydrive_shared_result_permission_none = (
+ (self.state.suspended_google_uids - uids_to_reactivate)
+ & self.state.permission_none_mydrive_shared_result_uids
+ & self.state.cancelled_twenty_four_months_ago_uids
+ ) - uids_to_scan_mydrive
+
+ # Form a set of all uids who have already been suspended, have a mydrive scan result
+ # `permissions-removed` and have been suspended in lookup for more than 24 months:
+ uids_to_delete_with_mydrive_shared_result_permission_removed = set()
+ if self.delete_users:
+ LOG.warning(
+ f"Excluding uids with mydrive-shared-result {MYDRIVE_SHARED_RESULT_REMOVED}."
+ )
+ #
+ # WARNING: There is a 6 months lead time ahead of this change, do
+ # do NOT enable the below code before September 2023.
+ #
+ # uids_to_delete_with_mydrive_shared_result_permission_removed = (
+ # (self.state.suspended_google_uids - uids_to_reactivate)
+ # & self.state.permission_removed_mydrive_shared_result_uids
+ # & self.state.cancelled_twenty_four_months_ago_uids
+ # ) - uids_to_scan_mydrive
+
+ # Form a super set of uids to delete
+ uids_to_delete = (
+ uids_to_delete_who_never_logged_in
+ | uids_to_delete_with_mydrive_shared_result_permission_none
+ | uids_to_delete_with_mydrive_shared_result_permission_removed
+ )
+
+ LOG.info("Number of users to suspend: %s", len(uids_to_suspend))
+ LOG.info(
+ f"Number of users to delete with my-shared-result {MYDRIVE_SHARED_RESULT_NONE}:"
+ f" {len(uids_to_delete_with_mydrive_shared_result_permission_none)}"
+ )
LOG.info(
- 'Number of users to delete instead of suspend: %s',
- len(delete_instead))
- LOG.info('Number of users to delete: %s', len(uids_to_delete))
- LOG.info('Number of users that need MyDrive scanned: %s', len(uids_to_scan_mydrive))
-
- self.state.update({
- 'google_user_updates': google_user_updates,
- 'google_user_creations': google_user_creations,
- 'uids_to_update': uids_to_update,
- 'uids_to_add': uids_to_add,
- 'uids_to_reactivate': uids_to_reactivate,
- 'uids_to_suspend': uids_to_suspend,
- 'uids_to_delete': uids_to_delete,
- 'uids_to_restore': uids_to_restore,
- 'uids_to_scan_mydrive': uids_to_scan_mydrive,
- })
+ f"Number of users to delete with my-shared-result {MYDRIVE_SHARED_RESULT_REMOVED}:"
+ f" {len(uids_to_delete_with_mydrive_shared_result_permission_removed)}"
+ )
+ LOG.info("Number of users to delete: %s", len(uids_to_delete))
+ LOG.info("Number of users that need MyDrive scanned: %s", len(uids_to_scan_mydrive))
+
+ self.state.update(
+ {
+ "google_user_updates": google_user_updates,
+ "google_user_creations": google_user_creations,
+ "uids_to_update": uids_to_update,
+ "uids_to_add": uids_to_add,
+ "uids_to_reactivate": uids_to_reactivate,
+ "uids_to_suspend": uids_to_suspend,
+ "uids_to_delete": uids_to_delete,
+ "uids_to_restore": uids_to_restore,
+ "uids_to_scan_mydrive": uids_to_scan_mydrive,
+ }
+ )
def compare_licensing(self):
# Compare Lookup and Googles licensed uids
if self.licensing_config.product_id is None:
- LOG.info('Skipping licensing assignment comparison as no Product Id set')
+ LOG.info("Skipping licensing assignment comparison as no Product Id set")
return
# Determining what to do depending on set membership:
@@ -167,16 +213,14 @@ class Comparator(ConfigurationStateConsumer):
# Those licensed in Google but not in Lookup need removing but ignore those eligible
# but not managed
- licences_to_remove = (
- (self.state.google_licensed_uids - self.state.licensed_uids)
- -
- (self.state.eligible_uids - self.state.managed_user_uids)
+ licences_to_remove = (self.state.google_licensed_uids - self.state.licensed_uids) - (
+ self.state.eligible_uids - self.state.managed_user_uids
)
- LOG.info('Number of licence assignments to remove: %s', len(licences_to_remove))
+ LOG.info("Number of licence assignments to remove: %s", len(licences_to_remove))
# Those licensed in Lookup but not in Google need adding
licences_to_add = self.state.licensed_uids - self.state.google_licensed_uids
- LOG.info('Number of licence assignments to add: %s', len(licences_to_add))
+ LOG.info("Number of licence assignments to add: %s", len(licences_to_add))
# Work out how many licences we will have available after updating
post_avail = (
@@ -184,16 +228,18 @@ class Comparator(ConfigurationStateConsumer):
+ len(licences_to_remove)
- len(licences_to_add)
)
- LOG.info('Number of licences available after update: %s', post_avail)
- self.state.update({
- 'licences_to_remove': licences_to_remove,
- 'licences_to_add': licences_to_add,
- })
+ LOG.info("Number of licences available after update: %s", post_avail)
+ self.state.update(
+ {
+ "licences_to_remove": licences_to_remove,
+ "licences_to_add": licences_to_add,
+ }
+ )
def compare_groups(self):
# For each group which exists in Google or the managed group set which is eligible,
# determine if it needs updating/creating. If so, record a patch/insert for the group.
- LOG.info('Calculating updates to groups...')
+ LOG.info("Calculating updates to groups...")
google_group_updates = {}
google_group_creations = {}
for gid, managed_group_entry in self.state.managed_group_entries_by_gid.items():
@@ -209,14 +255,14 @@ class Comparator(ConfigurationStateConsumer):
# Lookup institutions, which is useful since some institution names do not fit in the
# Google name field.
expected_google_group = {
- 'name': _trim_text(
- managed_group_entry.groupName, maxlen=73,
- suffix=self.sync_config.group_name_suffix
+ "name": _trim_text(
+ managed_group_entry.groupName,
+ maxlen=73,
+ suffix=self.sync_config.group_name_suffix,
+ ),
+ "description": _trim_text(
+ _clean_group_desc(managed_group_entry.description), maxlen=300
),
- 'description': _trim_text(
- _clean_group_desc(managed_group_entry.description),
- maxlen=300
- )
}
# Find existing Google group (if any).
@@ -227,11 +273,13 @@ class Comparator(ConfigurationStateConsumer):
# Unless anything needs changing, the patch is empty.
patch = {}
- if existing_google_group.get('name') != expected_google_group['name']:
- patch['name'] = expected_google_group['name']
- if (existing_google_group.get('description') !=
- expected_google_group['description']):
- patch['description'] = expected_google_group['description']
+ if existing_google_group.get("name") != expected_google_group["name"]:
+ patch["name"] = expected_google_group["name"]
+ if (
+ existing_google_group.get("description")
+ != expected_google_group["description"]
+ ):
+ patch["description"] = expected_google_group["description"]
# Only record non-empty patches.
if len(patch) > 0:
@@ -239,21 +287,21 @@ class Comparator(ConfigurationStateConsumer):
else:
# No existing Google group, so create one.
google_group_creations[gid] = {
- 'email': gid_to_email(gid, self.state.groups_domain, self.state.insts_domain),
- **expected_google_group
+ "email": gid_to_email(gid, self.state.groups_domain, self.state.insts_domain),
+ **expected_google_group,
}
# Form a set of all the gids which need patching.
gids_to_update = set(google_group_updates.keys())
- LOG.info('Number of existing groups to update: %s', len(gids_to_update))
+ LOG.info("Number of existing groups to update: %s", len(gids_to_update))
# Form a set of all the gids which need adding.
gids_to_add = set(google_group_creations.keys())
- LOG.info('Number of groups to add: %s', len(gids_to_add))
+ LOG.info("Number of groups to add: %s", len(gids_to_add))
# Form a set of all gids which need deleting.
gids_to_delete = self.state.all_google_gids - self.state.eligible_gids
- LOG.info('Number of groups to delete: %s', len(gids_to_delete))
+ LOG.info("Number of groups to delete: %s", len(gids_to_delete))
# For each managed group, determine which members to insert or delete. These are lists of
# (gid, uid) tuples.
@@ -263,56 +311,59 @@ class Comparator(ConfigurationStateConsumer):
# Find the existing Google group members.
existing_google_group = self.state.all_google_groups_by_gid.get(gid)
if existing_google_group:
- existing_members = self.state.all_google_members[existing_google_group['id']]
- existing_member_uids = set([email_to_uid(m['email']) for m in existing_members])
+ existing_members = self.state.all_google_members[existing_google_group["id"]]
+ existing_member_uids = set([email_to_uid(m["email"]) for m in existing_members])
else:
existing_member_uids = set()
# Members to insert. This is restricted to the managed user set, so that we don't
# attempt to insert a member resource for a non-existent user.
- insert_uids = (
- (managed_group_entry.uids - existing_member_uids)
- .intersection(self.state.managed_user_uids)
+ insert_uids = (managed_group_entry.uids - existing_member_uids).intersection(
+ self.state.managed_user_uids
)
members_to_insert.extend([(gid, uid) for uid in insert_uids])
# Members to delete. This will delete a member resource when the user is suspended
# and so we they will get re-added to it if the user is reactivated
- delete_uids = (existing_member_uids - managed_group_entry.uids)
+ delete_uids = existing_member_uids - managed_group_entry.uids
members_to_delete.extend([(gid, uid) for uid in delete_uids])
- LOG.info('Number of group members to insert: %s', len(members_to_insert))
- LOG.info('Number of group members to delete: %s', len(members_to_delete))
-
- self.state.update({
- 'google_group_updates': google_group_updates,
- 'google_group_creations': google_group_creations,
- 'gids_to_update': gids_to_update,
- 'gids_to_add': gids_to_add,
- 'gids_to_delete': gids_to_delete,
- 'members_to_insert': members_to_insert,
- 'members_to_delete': members_to_delete,
- })
+ LOG.info("Number of group members to insert: %s", len(members_to_insert))
+ LOG.info("Number of group members to delete: %s", len(members_to_delete))
+
+ self.state.update(
+ {
+ "google_group_updates": google_group_updates,
+ "google_group_creations": google_group_creations,
+ "gids_to_update": gids_to_update,
+ "gids_to_add": gids_to_add,
+ "gids_to_delete": gids_to_delete,
+ "members_to_insert": members_to_insert,
+ "members_to_delete": members_to_delete,
+ }
+ )
def compare_groups_settings(self):
# Determine changes to existing group settings
group_settings_to_update = {}
for gid, settings in self.state.all_google_group_settings_by_gid.items():
patch = {
- k: v for k, v in self.sync_config.group_settings.items()
- if settings.get(k) != v
+ k: v for k, v in self.sync_config.group_settings.items() if settings.get(k) != v
}
if len(patch) > 0:
group_settings_to_update[gid] = patch
gids_to_update_group_settings = set(group_settings_to_update.keys())
- LOG.info('Number of existing groups to update settings: %s',
- len(gids_to_update_group_settings))
+ LOG.info(
+ "Number of existing groups to update settings: %s", len(gids_to_update_group_settings)
+ )
- self.state.update({
- 'group_settings_to_update': group_settings_to_update,
- 'gids_to_update_group_settings': gids_to_update_group_settings,
- })
+ self.state.update(
+ {
+ "group_settings_to_update": group_settings_to_update,
+ "gids_to_update_group_settings": gids_to_update_group_settings,
+ }
+ )
def enforce_limits(self, just_users, licensing):
# --------------------------------------------------------------------------------------------
@@ -320,83 +371,97 @@ class Comparator(ConfigurationStateConsumer):
# --------------------------------------------------------------------------------------------
# Calculate percentage change to users, groups and group members.
- user_change_percentage = 100. * (
- len(self.state.uids_to_add | self.state.uids_to_update |
- self.state.uids_to_reactivate | self.state.uids_to_suspend |
- self.state.uids_to_delete | self.state.uids_to_restore |
- self.state.uids_to_scan_mydrive)
- /
- max(1, len(self.state.all_google_uids))
+ user_change_percentage = 100.0 * (
+ len(
+ self.state.uids_to_add
+ | self.state.uids_to_update
+ | self.state.uids_to_reactivate
+ | self.state.uids_to_suspend
+ | self.state.uids_to_delete
+ | self.state.uids_to_restore
+ | self.state.uids_to_scan_mydrive
+ )
+ / max(1, len(self.state.all_google_uids))
)
- LOG.info('Configuration will modify %.2f%% of users', user_change_percentage)
+ LOG.info("Configuration will modify %.2f%% of users", user_change_percentage)
if licensing and self.licensing_config.product_id is not None:
# Calculate percentage change in licence assignments.
- licence_change_percentage = 100. * (
+ licence_change_percentage = 100.0 * (
len(self.state.licences_to_add | self.state.licences_to_remove)
- /
- max(1, len(self.state.google_licensed_uids))
+ / max(1, len(self.state.google_licensed_uids))
)
LOG.info(
- 'Configuration will modify %.2f%% of licence assignments',
- licence_change_percentage)
+ "Configuration will modify %.2f%% of licence assignments",
+ licence_change_percentage,
+ )
if not just_users:
- group_change_percentage = 100. * (
+ group_change_percentage = 100.0 * (
len(self.state.gids_to_add | self.state.gids_to_update | self.state.gids_to_delete)
- /
- max(1, len(self.state.all_google_gids))
+ / max(1, len(self.state.all_google_gids))
)
- LOG.info('Configuration will modify %.2f%% of groups', group_change_percentage)
+ LOG.info("Configuration will modify %.2f%% of groups", group_change_percentage)
- member_change_percentage = 100. * (
+ member_change_percentage = 100.0 * (
(len(self.state.members_to_insert) + len(self.state.members_to_delete))
- /
- max(1, sum([len(m) for g, m in self.state.all_google_members.items()]))
+ / max(1, sum([len(m) for g, m in self.state.all_google_members.items()]))
)
- LOG.info('Configuration will modify %.2f%% of group members', member_change_percentage)
+ LOG.info("Configuration will modify %.2f%% of group members", member_change_percentage)
# Enforce percentage change sanity checks.
- if (self.limits_config.abort_user_change_percentage is not None and
- user_change_percentage > self.limits_config.abort_user_change_percentage):
+ if (
+ self.limits_config.abort_user_change_percentage is not None
+ and user_change_percentage > self.limits_config.abort_user_change_percentage
+ ):
LOG.error(
- 'Modification of %.2f%% of users is greater than limit of %.2f%%. Aborting.',
- user_change_percentage, self.limits_config.abort_user_change_percentage
+ "Modification of %.2f%% of users is greater than limit of %.2f%%. Aborting.",
+ user_change_percentage,
+ self.limits_config.abort_user_change_percentage,
)
- raise RuntimeError('Aborting due to large user change percentage')
+ raise RuntimeError("Aborting due to large user change percentage")
if licensing and self.licensing_config.product_id is not None:
if (
- self.limits_config.abort_licence_change_percentage is not None and
- licence_change_percentage > self.limits_config.abort_licence_change_percentage
+ self.limits_config.abort_licence_change_percentage is not None
+ and licence_change_percentage > self.limits_config.abort_licence_change_percentage
):
LOG.error(
- 'Modification of %.2f%% of licences is greater than limit of %.2f%%. '
- 'Aborting.',
- licence_change_percentage, self.limits_config.abort_licence_change_percentage
+ "Modification of %.2f%% of licences is greater than limit of %.2f%%. "
+ "Aborting.",
+ licence_change_percentage,
+ self.limits_config.abort_licence_change_percentage,
)
- raise RuntimeError('Aborting due to large licence assignment change percentage')
+ raise RuntimeError("Aborting due to large licence assignment change percentage")
if not just_users:
- if (self.limits_config.abort_group_change_percentage is not None and
- group_change_percentage > self.limits_config.abort_group_change_percentage):
+ if (
+ self.limits_config.abort_group_change_percentage is not None
+ and group_change_percentage > self.limits_config.abort_group_change_percentage
+ ):
LOG.error(
- 'Modification of %.2f%% of groups is greater than limit of %.2f%%. Aborting.',
- group_change_percentage, self.limits_config.abort_group_change_percentage
+ "Modification of %.2f%% of groups is greater than limit of %.2f%%. Aborting.",
+ group_change_percentage,
+ self.limits_config.abort_group_change_percentage,
)
- raise RuntimeError('Aborting due to large group change percentage')
- if (self.limits_config.abort_member_change_percentage is not None and
- member_change_percentage > self.limits_config.abort_member_change_percentage):
+ raise RuntimeError("Aborting due to large group change percentage")
+ if (
+ self.limits_config.abort_member_change_percentage is not None
+ and member_change_percentage > self.limits_config.abort_member_change_percentage
+ ):
LOG.error(
- 'Modification of %.2f%% of group members is greater than limit of %.2f%%. '
- 'Aborting.',
- member_change_percentage, self.limits_config.abort_member_change_percentage
+ "Modification of %.2f%% of group members is greater than limit of %.2f%%. "
+ "Aborting.",
+ member_change_percentage,
+ self.limits_config.abort_member_change_percentage,
)
- raise RuntimeError('Aborting due to large group member change percentage')
+ raise RuntimeError("Aborting due to large group member change percentage")
# Cap maximum size of various operations.
- if (self.limits_config.max_new_users is not None
- and len(self.state.uids_to_add) > self.limits_config.max_new_users):
+ if (
+ self.limits_config.max_new_users is not None
+ and len(self.state.uids_to_add) > self.limits_config.max_new_users
+ ):
# Ensure that we do not attempt to insert a group member for any of the users not
# added as a result of this cap, since these users won't exist in Google
capped_uids_to_add = _limit(self.state.uids_to_add, self.limits_config.max_new_users)
@@ -406,77 +471,92 @@ class Comparator(ConfigurationStateConsumer):
(g, u) for g, u in self.state.members_to_insert if u not in uids_not_added
]
self.state.uids_to_add = capped_uids_to_add
- LOG.info('Capped number of new users to %s', len(self.state.uids_to_add))
+ LOG.info("Capped number of new users to %s", len(self.state.uids_to_add))
- if (self.limits_config.max_suspended_users is not None and
- len(self.state.uids_to_suspend) > self.limits_config.max_suspended_users):
+ if (
+ self.limits_config.max_suspended_users is not None
+ and len(self.state.uids_to_suspend) > self.limits_config.max_suspended_users
+ ):
self.state.uids_to_suspend = _limit(
self.state.uids_to_suspend, self.limits_config.max_suspended_users
)
- LOG.info('Capped number of users to suspend to %s', len(self.state.uids_to_suspend))
- if (self.limits_config.max_deleted_users is not None and
- len(self.state.uids_to_delete) > self.limits_config.max_deleted_users):
+ LOG.info("Capped number of users to suspend to %s", len(self.state.uids_to_suspend))
+ if (
+ self.limits_config.max_deleted_users is not None
+ and len(self.state.uids_to_delete) > self.limits_config.max_deleted_users
+ ):
self.state.uids_to_delete = _limit(
self.state.uids_to_delete, self.limits_config.max_deleted_users
)
- LOG.info('Capped number of users to delete to %s', len(self.state.uids_to_delete))
- if (self.limits_config.max_restored_users is not None and
- len(self.state.uids_to_restore) > self.limits_config.max_restored_users):
+ LOG.info("Capped number of users to delete to %s", len(self.state.uids_to_delete))
+ if (
+ self.limits_config.max_restored_users is not None
+ and len(self.state.uids_to_restore) > self.limits_config.max_restored_users
+ ):
self.state.uids_to_restore = _limit(
self.state.uids_to_restore, self.limits_config.max_restored_users
)
LOG.info(
- 'Capped number of deleted users to restore to %s',
- len(self.state.uids_to_restore))
- if (self.limits_config.max_reactivated_users is not None and
- len(self.state.uids_to_reactivate) > self.limits_config.max_reactivated_users):
+ "Capped number of deleted users to restore to %s", len(self.state.uids_to_restore)
+ )
+ if (
+ self.limits_config.max_reactivated_users is not None
+ and len(self.state.uids_to_reactivate) > self.limits_config.max_reactivated_users
+ ):
self.state.uids_to_reactivate = _limit(
self.state.uids_to_reactivate, self.limits_config.max_reactivated_users
)
LOG.info(
- 'Capped number of users to reactivate to %s',
- len(self.state.uids_to_reactivate)
+ "Capped number of users to reactivate to %s", len(self.state.uids_to_reactivate)
)
- if (self.limits_config.max_mydrive_scan_users is not None and
- len(self.state.uids_to_scan_mydrive) > self.limits_config.max_mydrive_scan_users):
+ if (
+ self.limits_config.max_mydrive_scan_users is not None
+ and len(self.state.uids_to_scan_mydrive) > self.limits_config.max_mydrive_scan_users
+ ):
self.state.uids_to_scan_mydrive = _limit(
self.state.uids_to_scan_mydrive, self.limits_config.max_mydrive_scan_users
)
LOG.info(
- 'Capped number of users to mark Mydrive to be scanned to %s',
- len(self.state.uids_to_scan_mydrive)
+ "Capped number of users to mark Mydrive to be scanned to %s",
+ len(self.state.uids_to_scan_mydrive),
)
- if (self.limits_config.max_updated_users is not None and
- len(self.state.uids_to_update) > self.limits_config.max_updated_users):
+ if (
+ self.limits_config.max_updated_users is not None
+ and len(self.state.uids_to_update) > self.limits_config.max_updated_users
+ ):
self.state.uids_to_update = _limit(
self.state.uids_to_update, self.limits_config.max_updated_users
)
- LOG.info('Capped number of users to update to %s', len(self.state.uids_to_update))
+ LOG.info("Capped number of users to update to %s", len(self.state.uids_to_update))
# Licences
if licensing and self.licensing_config.product_id is not None:
- if (self.limits_config.max_add_licence is not None and
- len(self.state.licences_to_add) > self.limits_config.max_add_licence):
+ if (
+ self.limits_config.max_add_licence is not None
+ and len(self.state.licences_to_add) > self.limits_config.max_add_licence
+ ):
self.state.licences_to_add = _limit(
self.state.licences_to_add, self.limits_config.max_add_licence
)
LOG.info(
- 'Capped number of licences to assign to %s',
- len(self.state.licences_to_add)
+ "Capped number of licences to assign to %s", len(self.state.licences_to_add)
)
- if (self.limits_config.max_remove_licence is not None and
- len(self.state.licences_to_remove) > self.limits_config.max_remove_licence):
+ if (
+ self.limits_config.max_remove_licence is not None
+ and len(self.state.licences_to_remove) > self.limits_config.max_remove_licence
+ ):
self.state.licences_to_remove = _limit(
self.state.licences_to_remove, self.limits_config.max_remove_licence
)
LOG.info(
- 'Capped number of licences to remove to %s',
- len(self.state.licences_to_remove)
+ "Capped number of licences to remove to %s", len(self.state.licences_to_remove)
)
if not just_users:
- if (self.limits_config.max_new_groups is not None and
- len(self.state.gids_to_add) > self.limits_config.max_new_groups):
+ if (
+ self.limits_config.max_new_groups is not None
+ and len(self.state.gids_to_add) > self.limits_config.max_new_groups
+ ):
# Ensure that we do not attempt to insert a group member for any of the groups not
# added as a result of this cap, since these groups won't exist in Google
capped_gids_to_add = _limit(
@@ -487,37 +567,45 @@ class Comparator(ConfigurationStateConsumer):
(g, u) for g, u in self.state.members_to_insert if g not in gids_not_added
]
self.state.gids_to_add = capped_gids_to_add
- LOG.info('Capped number of new groups to %s', len(self.state.gids_to_add))
+ LOG.info("Capped number of new groups to %s", len(self.state.gids_to_add))
- if (self.limits_config.max_deleted_groups is not None and
- len(self.state.gids_to_delete) > self.limits_config.max_deleted_groups):
+ if (
+ self.limits_config.max_deleted_groups is not None
+ and len(self.state.gids_to_delete) > self.limits_config.max_deleted_groups
+ ):
self.state.gids_to_delete = _limit(
self.state.gids_to_delete, self.limits_config.max_deleted_groups
)
- LOG.info('Capped number of groups to delete to %s', len(self.state.gids_to_delete))
- if (self.limits_config.max_updated_groups is not None and
- len(self.state.gids_to_update) > self.limits_config.max_updated_groups):
+ LOG.info("Capped number of groups to delete to %s", len(self.state.gids_to_delete))
+ if (
+ self.limits_config.max_updated_groups is not None
+ and len(self.state.gids_to_update) > self.limits_config.max_updated_groups
+ ):
self.state.gids_to_update = _limit(
self.state.gids_to_update, self.limits_config.max_updated_groups
)
- LOG.info('Capped number of groups to update to %s', len(self.state.gids_to_update))
- if (self.limits_config.max_inserted_members is not None and
- len(self.state.members_to_insert) > self.limits_config.max_inserted_members):
- self.state.members_to_insert = (
- self.state.members_to_insert[0:self.limits_config.max_inserted_members]
- )
+ LOG.info("Capped number of groups to update to %s", len(self.state.gids_to_update))
+ if (
+ self.limits_config.max_inserted_members is not None
+ and len(self.state.members_to_insert) > self.limits_config.max_inserted_members
+ ):
+ self.state.members_to_insert = self.state.members_to_insert[
+ 0 : self.limits_config.max_inserted_members
+ ]
LOG.info(
- 'Capped number of group members to insert to %s',
- len(self.state.members_to_insert)
- )
- if (self.limits_config.max_deleted_members is not None and
- len(self.state.members_to_delete) > self.limits_config.max_deleted_members):
- self.state.members_to_delete = (
- self.state.members_to_delete[0:self.limits_config.max_deleted_members]
+ "Capped number of group members to insert to %s",
+ len(self.state.members_to_insert),
)
+ if (
+ self.limits_config.max_deleted_members is not None
+ and len(self.state.members_to_delete) > self.limits_config.max_deleted_members
+ ):
+ self.state.members_to_delete = self.state.members_to_delete[
+ 0 : self.limits_config.max_deleted_members
+ ]
LOG.info(
- 'Capped number of group members to delete to %s',
- len(self.state.members_to_delete)
+ "Capped number of group members to delete to %s",
+ len(self.state.members_to_delete),
)
@@ -546,7 +634,7 @@ def _limit(s, limit):
return {e for _, e in itertools.takewhile(lambda p: p[0] < limit, enumerate(s))}
-def _trim_text(text, *, maxlen, cont='...', suffix=''):
+def _trim_text(text, *, maxlen, cont="...", suffix=""):
"""
Trim text to be no more than "maxlen" characters long, terminating it with "cont" if it had
to be truncated. If supplied, "suffix" is appended to the string after truncating, and the
@@ -554,8 +642,9 @@ def _trim_text(text, *, maxlen, cont='...', suffix=''):
"""
return (
- text[0:maxlen-len(cont)-len(suffix)]+cont+suffix
- if len(text)+len(suffix) > maxlen else text+suffix
+ text[0 : maxlen - len(cont) - len(suffix)] + cont + suffix
+ if len(text) + len(suffix) > maxlen
+ else text + suffix
)
@@ -570,8 +659,8 @@ def _clean_group_desc(s):
'abcd'
"""
- return ''.join(c for c in s if c not in _CLEAN_GROUP_DESC_BAD_CHARS)
+ return "".join(c for c in s if c not in _CLEAN_GROUP_DESC_BAD_CHARS)
# Characters stripped by _clean_group_desc. Present as a constant to avoid re-creating it.
-_CLEAN_GROUP_DESC_BAD_CHARS = '=<>'
+_CLEAN_GROUP_DESC_BAD_CHARS = "=<>"
diff --git a/gsuitesync/sync/gapi.py b/gsuitesync/sync/gapi.py
index 36c1bc8..eac80cc 100644
--- a/gsuitesync/sync/gapi.py
+++ b/gsuitesync/sync/gapi.py
@@ -12,38 +12,50 @@ import socket
from .base import ConfigurationStateConsumer
from .. import gapiutil
-from .utils import (email_to_uid, email_to_gid, groupID_regex, instID_regex,
- date_months_ago, isodate_parse, custom_action, cache_to_disk)
+from .utils import (
+ email_to_uid,
+ email_to_gid,
+ groupID_regex,
+ instID_regex,
+ date_months_ago,
+ isodate_parse,
+ custom_action,
+ cache_to_disk,
+)
LOG = logging.getLogger(__name__)
# Scopes required to perform read-only actions.
READ_ONLY_SCOPES = [
- 'https://www.googleapis.com/auth/admin.directory.user.readonly',
- 'https://www.googleapis.com/auth/admin.directory.group.readonly',
- 'https://www.googleapis.com/auth/admin.directory.group.member.readonly',
- 'https://www.googleapis.com/auth/apps.groups.settings',
- 'https://www.googleapis.com/auth/apps.licensing',
+ "https://www.googleapis.com/auth/admin.directory.user.readonly",
+ "https://www.googleapis.com/auth/admin.directory.group.readonly",
+ "https://www.googleapis.com/auth/admin.directory.group.member.readonly",
+ "https://www.googleapis.com/auth/apps.groups.settings",
+ "https://www.googleapis.com/auth/apps.licensing",
]
# Scopes *in addition to READ_ONLY_SCOPES* required to perform a full update.
WRITE_SCOPES = [
- 'https://www.googleapis.com/auth/admin.directory.user',
- 'https://www.googleapis.com/auth/admin.directory.group',
- 'https://www.googleapis.com/auth/admin.directory.group.member',
+ "https://www.googleapis.com/auth/admin.directory.user",
+ "https://www.googleapis.com/auth/admin.directory.group",
+ "https://www.googleapis.com/auth/admin.directory.group.member",
]
+MYDRIVE_SHARED_ACTION_SCAN = "scan"
+MYDRIVE_SHARED_RESULT_NONE = "permissions-none"
+MYDRIVE_SHARED_RESULT_REMOVED = "permissions-removed"
+
class GAPIRetriever(ConfigurationStateConsumer):
- required_config = ('gapi_auth', 'gapi_domain', 'sync', 'licensing')
+ required_config = ("gapi_auth", "gapi_domain", "sync", "licensing")
def connect(self, timeout=300):
# load credentials
self.creds = self._get_credentials(self.read_only)
# Build the directory service using Google API discovery.
socket.setdefaulttimeout(timeout)
- directory_service = discovery.build('admin', 'directory_v1', credentials=self.creds)
+ directory_service = discovery.build("admin", "directory_v1", credentials=self.creds)
# Secondary domain for Google groups that come from Lookup groups
groups_domain = (
@@ -61,23 +73,21 @@ class GAPIRetriever(ConfigurationStateConsumer):
# We also build the groupssettings service, which is a parallel API to manage group
# settings. We do this even if not given `--group-settings` argument as newly created
# groups will need their settings updated.
- groupssettings_service = discovery.build(
- 'groupssettings', 'v1', credentials=self.creds
- )
+ groupssettings_service = discovery.build("groupssettings", "v1", credentials=self.creds)
# Also build the directory service for using the licensing API
- licensing_service = discovery.build(
- 'licensing', 'v1', credentials=self.creds
- )
+ licensing_service = discovery.build("licensing", "v1", credentials=self.creds)
# Return components needed for connection with Google API
- self.state.update({
- 'directory_service': directory_service,
- 'groupssettings_service': groupssettings_service,
- 'licensing_service': licensing_service,
- 'groups_domain': groups_domain,
- 'insts_domain': insts_domain,
- })
+ self.state.update(
+ {
+ "directory_service": directory_service,
+ "groupssettings_service": groupssettings_service,
+ "licensing_service": licensing_service,
+ "groups_domain": groups_domain,
+ "insts_domain": insts_domain,
+ }
+ )
def _get_credentials(self, read_only):
"""
@@ -89,7 +99,7 @@ class GAPIRetriever(ConfigurationStateConsumer):
creds_file = self.gapi_auth_config.credentials
if read_only and self.gapi_auth_config.read_only_credentials is not None:
creds = self.gapi_auth_config.read_only_credentials
- LOG.info('Using read-only credentials.')
+ LOG.info("Using read-only credentials.")
LOG.info('Loading Google account credentials from "%s"', creds_file)
creds = service_account.Credentials.from_service_account_file(creds_file)
@@ -108,40 +118,62 @@ class GAPIRetriever(ConfigurationStateConsumer):
all_google_users = self._filtered_user_list(show_deleted=False)
# Form mappings from uid to Google user.
- all_google_users_by_uid = {
- email_to_uid(u['primaryEmail']): u for u in all_google_users
- }
+ all_google_users_by_uid = {email_to_uid(u["primaryEmail"]): u for u in all_google_users}
# Form sets of all Google-side uids. The all_google_uids set is all users including
# the suspended ones and the suspended_google_uids set is only the suspended users. Non
# suspended users are therefore all_google_uids - suspended_google_uids.
all_google_uids = set(all_google_users_by_uid.keys())
suspended_google_uids = {
- uid for uid, u in all_google_users_by_uid.items() if u['suspended']
+ uid for uid, u in all_google_users_by_uid.items() if u["suspended"]
}
# Also form a set of all uids who have never logged in
# (lastLoginTime will be 1970-01-01T00:00:00.000Z)
never_logged_in_google_uids = {
- uid for uid, u in all_google_users_by_uid.items() if u['lastLoginTime'][:4] == '1970'
+ uid for uid, u in all_google_users_by_uid.items() if u["lastLoginTime"][:4] == "1970"
}
# And form a set of all suspended uids who have never logged in in last six months
six_months_ago = date_months_ago(6)
never_logged_in_six_months_uids = {
- uid for uid, u in all_google_users_by_uid.items()
- if u['suspended'] and u['lastLoginTime'] and
- isodate_parse(u['lastLoginTime']) < six_months_ago
+ uid
+ for uid, u in all_google_users_by_uid.items()
+ if u["suspended"]
+ and u["lastLoginTime"]
+ and isodate_parse(u["lastLoginTime"]) < six_months_ago
}
# Form a set of suspended uids that have no mydrive-shared-action or -result
no_mydrive_shared_settings_uids = {
- uid for uid, u in all_google_users_by_uid.items()
- if u['suspended'] and not custom_action(u, 'mydrive-shared-action') and
- not custom_action(u, 'mydrive-shared-result')
+ uid
+ for uid, u in all_google_users_by_uid.items()
+ if u["suspended"]
+ and not custom_action(u, "mydrive-shared-action")
+ and not custom_action(u, "mydrive-shared-result")
+ }
+
+ # Form a set of suspended uids that have no mydrive-shared-action and
+ # mydrive-shared-result is `permissions-none`
+ permission_none_mydrive_shared_result_uids = {
+ uid
+ for uid, u in all_google_users_by_uid.items()
+ if u["suspended"]
+ and not custom_action(u, "mydrive-shared-action")
+ and custom_action(u, "mydrive-shared-result") == MYDRIVE_SHARED_RESULT_NONE
+ }
+
+ # Form a set of suspended uids that have no mydrive-shared-action and
+ # mydrive-shared-result is `permissions-removed`
+ permission_removed_mydrive_shared_result_uids = {
+ uid
+ for uid, u in all_google_users_by_uid.items()
+ if u["suspended"]
+ and not custom_action(u, "mydrive-shared-action")
+ and custom_action(u, "mydrive-shared-result") == MYDRIVE_SHARED_RESULT_REMOVED
}
# Sanity check. We should not have lost anything. (I.e. the uids should be unique.)
if len(all_google_uids) != len(all_google_users):
- raise RuntimeError('Sanity check failed: user list changed length')
+ raise RuntimeError("Sanity check failed: user list changed length")
# Get recently deleted users so we can restore them instead of recreating a duplicate
# new account if needed
@@ -149,115 +181,140 @@ class GAPIRetriever(ConfigurationStateConsumer):
# There are potentially multiple deletions of the same user. Assuming we want to restore
# the last deleted, sort so that the dict references the last one.
- all_deleted_google_users.sort(key=lambda u: u['deletionTime'])
+ all_deleted_google_users.sort(key=lambda u: u["deletionTime"])
all_deleted_google_users_by_uid = {
- email_to_uid(u['primaryEmail']): u for u in all_deleted_google_users
+ email_to_uid(u["primaryEmail"]): u for u in all_deleted_google_users
}
all_deleted_google_uids = set(all_deleted_google_users_by_uid.keys())
# Log some stats.
- LOG.info('Total Google users: %s', len(all_google_uids))
- LOG.info('Suspended Google users: %s', len(suspended_google_uids))
- LOG.info('Never Logged-in Google users: %s', len(never_logged_in_google_uids))
- LOG.info('Not Logged-in for 6 months suspended Google users: %s',
- len(never_logged_in_six_months_uids))
- LOG.info('No MyDrive settings suspended Google users: %s',
- len(no_mydrive_shared_settings_uids))
- LOG.info('Recently Deleted Google users: %s', len(all_deleted_google_uids))
-
- self.state.update({
- 'all_google_users': all_google_users,
- 'all_google_users_by_uid': all_google_users_by_uid,
- 'all_google_uids': all_google_uids,
- 'suspended_google_uids': suspended_google_uids,
- 'never_logged_in_google_uids': never_logged_in_google_uids,
- 'never_logged_in_six_months_uids': never_logged_in_six_months_uids,
- 'no_mydrive_shared_settings_uids': no_mydrive_shared_settings_uids,
- 'all_deleted_google_users_by_uid': all_deleted_google_users_by_uid,
- })
-
- @cache_to_disk('google_users_{show_deleted}')
+ LOG.info("Total Google users: %s", len(all_google_uids))
+ LOG.info("Suspended Google users: %s", len(suspended_google_uids))
+ LOG.info("Never Logged-in Google users: %s", len(never_logged_in_google_uids))
+ LOG.info(
+ "Not Logged-in for 6 months suspended Google users: %s",
+ len(never_logged_in_six_months_uids),
+ )
+ LOG.info(
+ "No MyDrive settings suspended Google users: %s", len(no_mydrive_shared_settings_uids)
+ )
+ LOG.info(
+ f"MyDrive shared result {MYDRIVE_SHARED_RESULT_NONE}"
+ f" suspended Google users: {len(permission_none_mydrive_shared_result_uids)}"
+ )
+ LOG.info(
+ f"MyDrive shared result {MYDRIVE_SHARED_RESULT_REMOVED}"
+ f" suspended Google users: {len(permission_removed_mydrive_shared_result_uids)}"
+ )
+ LOG.info("Recently Deleted Google users: %s", len(all_deleted_google_uids))
+
+ self.state.update(
+ {
+ "all_google_users": all_google_users,
+ "all_google_users_by_uid": all_google_users_by_uid,
+ "all_google_uids": all_google_uids,
+ "suspended_google_uids": suspended_google_uids,
+ "never_logged_in_google_uids": never_logged_in_google_uids,
+ "never_logged_in_six_months_uids": never_logged_in_six_months_uids,
+ "no_mydrive_shared_settings_uids": no_mydrive_shared_settings_uids,
+ "permission_none_mydrive_shared_result_uids": permission_none_mydrive_shared_result_uids, # NOQA: E501
+ "permission_removed_mydrive_shared_result_uids": permission_removed_mydrive_shared_result_uids, # NOQA: E501
+ "all_deleted_google_users_by_uid": all_deleted_google_users_by_uid,
+ }
+ )
+
+ @cache_to_disk("google_users_{show_deleted}")
def _filtered_user_list(self, show_deleted=False):
LOG.info(
f"Getting information on {'deleted' if show_deleted else 'active'}"
- ' Google domain users')
+ " Google domain users"
+ )
fields = [
- 'id', 'isAdmin', 'orgUnitPath', 'primaryEmail', 'suspended', 'lastLoginTime',
- 'name(givenName, familyName)', 'customSchemas',
- ] + (['deletionTime'] if show_deleted else [])
+ "id",
+ "isAdmin",
+ "orgUnitPath",
+ "primaryEmail",
+ "suspended",
+ "lastLoginTime",
+ "name(givenName, familyName)",
+ "customSchemas",
+ ] + (["deletionTime"] if show_deleted else [])
all_google_users = gapiutil.list_all(
- self.state.directory_service.users().list, items_key='users',
- domain=self.gapi_domain_config.name, showDeleted=show_deleted,
- query='isAdmin=false', fields='nextPageToken,users(' + ','.join(fields) + ')',
- projection='custom', customFieldMask='UCam',
- retries=self.sync_config.http_retries, retry_delay=self.sync_config.http_retry_delay,
+ self.state.directory_service.users().list,
+ items_key="users",
+ domain=self.gapi_domain_config.name,
+ showDeleted=show_deleted,
+ query="isAdmin=false",
+ fields="nextPageToken,users(" + ",".join(fields) + ")",
+ projection="custom",
+ customFieldMask="UCam",
+ retries=self.sync_config.http_retries,
+ retry_delay=self.sync_config.http_retry_delay,
allow_empty=show_deleted,
)
# Strip any "to be ignored" users out of the results.
# Deleted users don't have an orgUnitPath.
if not show_deleted and self.sync_config.ignore_google_org_unit_path_regex is not None:
LOG.info(
- 'Ignoring users whose organization unit path matches %r',
- self.sync_config.ignore_google_org_unit_path_regex)
+ "Ignoring users whose organization unit path matches %r",
+ self.sync_config.ignore_google_org_unit_path_regex,
+ )
# Check that all users have an orgUnitPath
- missing_org = [
- u for u in all_google_users if 'orgUnitPath' not in u
- ]
+ missing_org = [u for u in all_google_users if "orgUnitPath" not in u]
if len(missing_org) != 0:
LOG.error(
- 'User entries missing orgUnitPath: %s (starting with %s)', len(missing_org),
- missing_org[0]['primaryEmail'] if 'primaryEmail' in missing_org[0]
- else 'user with blank email'
+ "User entries missing orgUnitPath: %s (starting with %s)",
+ len(missing_org),
+ missing_org[0]["primaryEmail"]
+ if "primaryEmail" in missing_org[0]
+ else "user with blank email",
)
- raise RuntimeError('Sanity check failed: at least one user is missing orgUnitPath')
+ raise RuntimeError("Sanity check failed: at least one user is missing orgUnitPath")
# Remove users matching regex
regex = re.compile(self.sync_config.ignore_google_org_unit_path_regex)
- all_google_users = [
- u for u in all_google_users if not regex.match(u['orgUnitPath'])
- ]
+ all_google_users = [u for u in all_google_users if not regex.match(u["orgUnitPath"])]
# Strip out any users with uids (extracted from the local-part of the email address) that
# aren't valid CRSids. These users can't have come from Lookup, and so should not be
# managed (suspended) by this script.
- all_google_users = [
- u for u in all_google_users if email_to_uid(u['primaryEmail'])
- ]
+ all_google_users = [u for u in all_google_users if email_to_uid(u["primaryEmail"])]
# Sanity check. There should be no admins in the returned results.
- if any(u.get('isAdmin', False) for u in all_google_users):
- raise RuntimeError('Sanity check failed: admin users in user list')
+ if any(u.get("isAdmin", False) for u in all_google_users):
+ raise RuntimeError("Sanity check failed: admin users in user list")
return all_google_users
def retrieve_licenses(self):
if self.licensing_config.product_id is None:
- LOG.info('Skipping licensing assignment as no Product Id set')
+ LOG.info("Skipping licensing assignment as no Product Id set")
return
# Retrieve all license assignments
- LOG.info('Getting information on licence assignment for Google domain users')
- LOG.info('Total licence assignments: %s', len(self.all_licence_assignments))
+ LOG.info("Getting information on licence assignment for Google domain users")
+ LOG.info("Total licence assignments: %s", len(self.all_licence_assignments))
# Build a map of uids to license SKU. We are only interested in those in our domain
# and SKUs in our configuration
skus_by_uid = {
- parts[1]: parts[0] for parts in
- [[lic['skuId']] + lic['userId'].split('@', 1) for lic in self.all_licence_assignments]
+ parts[1]: parts[0]
+ for parts in [
+ [lic["skuId"]] + lic["userId"].split("@", 1)
+ for lic in self.all_licence_assignments
+ ]
if (
- len(parts) == 3 and
- parts[2] == self.gapi_domain_config.name and
- parts[0] in self.licensing_config.licensed_skus.keys()
+ len(parts) == 3
+ and parts[2] == self.gapi_domain_config.name
+ and parts[0] in self.licensing_config.licensed_skus.keys()
)
}
# Build the set of licensed uids
licensed_uids = set(skus_by_uid.keys())
- LOG.info('Total licensed uids: %s', len(licensed_uids))
+ LOG.info("Total licensed uids: %s", len(licensed_uids))
# Count available licences left for each SKU
available_by_sku = {
- avail_sku: (total - len({
- uid for uid, sku in skus_by_uid.items() if sku == avail_sku
- }))
+ avail_sku: (total - len({uid for uid, sku in skus_by_uid.items() if sku == avail_sku}))
for avail_sku, total in self.licensing_config.licensed_skus.items()
}
for sku, available in available_by_sku.items():
@@ -266,48 +323,53 @@ class GAPIRetriever(ConfigurationStateConsumer):
total_available = sum(available_by_sku.values())
LOG.info(f"Licences available in total: {total_available}")
- self.state.update({
- 'google_licensed_uids': licensed_uids,
- 'google_skus_by_uid': skus_by_uid,
- 'google_available_by_sku': available_by_sku,
- })
+ self.state.update(
+ {
+ "google_licensed_uids": licensed_uids,
+ "google_skus_by_uid": skus_by_uid,
+ "google_available_by_sku": available_by_sku,
+ }
+ )
@functools.cached_property
- @cache_to_disk('google_licence_assignments')
+ @cache_to_disk("google_licence_assignments")
def all_licence_assignments(self):
- fields = ['userId', 'skuId']
+ fields = ["userId", "skuId"]
return gapiutil.list_all(
self.state.licensing_service.licenseAssignments().listForProduct,
customerId=self.licensing_config.customer_id,
productId=self.licensing_config.product_id,
- fields='nextPageToken,items(' + ','.join(fields) + ')',
- retries=self.sync_config.http_retries, retry_delay=self.sync_config.http_retry_delay,
+ fields="nextPageToken,items(" + ",".join(fields) + ")",
+ retries=self.sync_config.http_retries,
+ retry_delay=self.sync_config.http_retry_delay,
)
def retrieve_groups(self):
# Retrieve information on all Google groups that come from Lookup groups
- LOG.info('Getting information on Google domain groups')
+ LOG.info("Getting information on Google domain groups")
all_google_groups = [
- g for g in self._fetch_groups(domain=self.state.groups_domain)
- if groupID_regex.match(g['email'].split('@')[0])
+ g
+ for g in self._fetch_groups(domain=self.state.groups_domain)
+ if groupID_regex.match(g["email"].split("@")[0])
]
# Append information on all Google groups that come from Lookup institutions
- LOG.info('Getting information on Google domain institutions')
- all_google_groups.extend([
- g for g in self._fetch_groups(domain=self.state.insts_domain)
- if instID_regex.match(g['email'].split('@')[0].upper())
- ])
+ LOG.info("Getting information on Google domain institutions")
+ all_google_groups.extend(
+ [
+ g
+ for g in self._fetch_groups(domain=self.state.insts_domain)
+ if instID_regex.match(g["email"].split("@")[0].upper())
+ ]
+ )
# Strip out any groups whose email addresses don't match the pattern for groups created
# from Lookup groupIDs or instIDs, and which therefore should not be managed (deleted) by
# this script.
- all_google_groups = [g for g in all_google_groups if email_to_gid(g['email'])]
+ all_google_groups = [g for g in all_google_groups if email_to_gid(g["email"])]
# Form mappings from gid to Google group.
- all_google_groups_by_gid = {
- email_to_gid(g['email']): g for g in all_google_groups
- }
+ all_google_groups_by_gid = {email_to_gid(g["email"]): g for g in all_google_groups}
# Form sets of all Google-side gids. The all_google_gids set includes both groupIDs and
# instIDs. Groups in Google do not have any concept of being suspended.
@@ -315,81 +377,96 @@ class GAPIRetriever(ConfigurationStateConsumer):
# Sanity check. We should not have lost anything. (I.e. the gids should be unique.)
if len(all_google_gids) != len(all_google_groups):
- raise RuntimeError('Sanity check failed: group list changed length')
+ raise RuntimeError("Sanity check failed: group list changed length")
# Retrieve all Google group memberships. This is a mapping from internal Google group ids
# to lists of member resources, corresponding to both Lookup groups and institutions.
- all_google_members = self._fetch_group_members([g['id'] for g in all_google_groups])
+ all_google_members = self._fetch_group_members([g["id"] for g in all_google_groups])
# Sanity check. We should have a group members list for each managed group.
if len(all_google_members) != len(all_google_groups):
raise RuntimeError(
- 'Sanity check failed: groups in members map do not match group list')
+ "Sanity check failed: groups in members map do not match group list"
+ )
# Log some stats.
- LOG.info('Total Google groups: %s', len(all_google_gids))
+ LOG.info("Total Google groups: %s", len(all_google_gids))
LOG.info(
- 'Total Google group members: %s',
- sum([len(m) for g, m in all_google_members.items()])
+ "Total Google group members: %s", sum([len(m) for g, m in all_google_members.items()])
)
- self.state.update({
- 'all_google_groups': all_google_groups,
- 'all_google_groups_by_gid': all_google_groups_by_gid,
- 'all_google_gids': all_google_gids,
- 'all_google_members': all_google_members,
- })
+ self.state.update(
+ {
+ "all_google_groups": all_google_groups,
+ "all_google_groups_by_gid": all_google_groups_by_gid,
+ "all_google_gids": all_google_gids,
+ "all_google_members": all_google_members,
+ }
+ )
def retrieve_group_settings(self):
# Retrieve all Google group settings.
- all_google_group_emails = [g['email'] for g in self.state.all_google_groups]
+ all_google_group_emails = [g["email"] for g in self.state.all_google_groups]
all_google_group_settings = self._fetch_group_settings(all_google_group_emails)
# Form a mapping from gid to Google group settings.
all_google_group_settings_by_gid = {
- email_to_gid(g['email']): g for g in all_google_group_settings
+ email_to_gid(g["email"]): g for g in all_google_group_settings
}
# Sanity check. We should have settings for each managed group.
if len(all_google_group_settings_by_gid) != len(self.state.all_google_groups):
raise RuntimeError(
- 'Sanity check failed: group settings list does not match group list'
+ "Sanity check failed: group settings list does not match group list"
)
- self.state.update({
- 'all_google_group_settings_by_gid': all_google_group_settings_by_gid,
- })
+ self.state.update(
+ {
+ "all_google_group_settings_by_gid": all_google_group_settings_by_gid,
+ }
+ )
- @cache_to_disk('google_groups_{domain}')
+ @cache_to_disk("google_groups_{domain}")
def _fetch_groups(self, domain):
"""
Function to fetch Google group information from the specified domain
"""
- fields = ['id', 'email', 'name', 'description']
+ fields = ["id", "email", "name", "description"]
return gapiutil.list_all(
- self.state.directory_service.groups().list, items_key='groups', domain=domain,
- fields='nextPageToken,groups(' + ','.join(fields) + ')',
- retries=self.sync_config.http_retries, retry_delay=self.sync_config.http_retry_delay,
+ self.state.directory_service.groups().list,
+ items_key="groups",
+ domain=domain,
+ fields="nextPageToken,groups(" + ",".join(fields) + ")",
+ retries=self.sync_config.http_retries,
+ retry_delay=self.sync_config.http_retry_delay,
)
- @cache_to_disk('google_group_members')
+ @cache_to_disk("google_group_members")
def _fetch_group_members(self, ids):
- fields = ['id', 'email']
+ fields = ["id", "email"]
return gapiutil.list_all_in_list(
- self.state.directory_service, self.state.directory_service.members().list,
- item_ids=ids, id_key='groupKey',
- batch_size=self.sync_config.batch_size, items_key='members',
- fields='nextPageToken,members(' + ','.join(fields) + ')',
- retries=self.sync_config.http_retries, retry_delay=self.sync_config.http_retry_delay,
+ self.state.directory_service,
+ self.state.directory_service.members().list,
+ item_ids=ids,
+ id_key="groupKey",
+ batch_size=self.sync_config.batch_size,
+ items_key="members",
+ fields="nextPageToken,members(" + ",".join(fields) + ")",
+ retries=self.sync_config.http_retries,
+ retry_delay=self.sync_config.http_retry_delay,
)
- @cache_to_disk('google_group_settings')
+ @cache_to_disk("google_group_settings")
def _fetch_group_settings(self, ids):
- fields = ['email', *[k for k in self.sync_config.group_settings.keys()]]
+ fields = ["email", *[k for k in self.sync_config.group_settings.keys()]]
return gapiutil.get_all_in_list(
- self.state.groupssettings_service, self.state.groupssettings_service.groups().get,
- item_ids=ids, id_key='groupUniqueId',
- batch_size=self.sync_config.batch_size, fields=','.join(fields),
- retries=self.sync_config.http_retries, retry_delay=self.sync_config.http_retry_delay,
+ self.state.groupssettings_service,
+ self.state.groupssettings_service.groups().get,
+ item_ids=ids,
+ id_key="groupUniqueId",
+ batch_size=self.sync_config.batch_size,
+ fields=",".join(fields),
+ retries=self.sync_config.http_retries,
+ retry_delay=self.sync_config.http_retry_delay,
)
diff --git a/gsuitesync/sync/lookup.py b/gsuitesync/sync/lookup.py
index 11dc09b..7c314e5 100644
--- a/gsuitesync/sync/lookup.py
+++ b/gsuitesync/sync/lookup.py
@@ -5,18 +5,16 @@ Load current user, group and institution data from Lookup.
import collections
import functools
import logging
-import yaml
-from identitylib.lookup_client_configuration import LookupClientConfiguration
+import yaml
from identitylib.lookup_client import ApiClient as LookupApiClient
-from identitylib.lookup_client.api.person_api import PersonApi
from identitylib.lookup_client.api.group_api import GroupApi
from identitylib.lookup_client.api.institution_api import InstitutionApi
-
-from .utils import date_months_ago, isodate_parse, cache_to_disk
+from identitylib.lookup_client.api.person_api import PersonApi
+from identitylib.lookup_client_configuration import LookupClientConfiguration
from .base import ConfigurationStateConsumer
-
+from .utils import cache_to_disk, date_months_ago, isodate_parse
LOG = logging.getLogger(__name__)
@@ -24,27 +22,30 @@ LOG = logging.getLogger(__name__)
DEFAULT_LIST_FETCH_LIMIT = 1000
# The scheme used to identify users
-UID_SCHEME = 'crsid'
+UID_SCHEME = "crsid"
# Extra attributes to fetch when checking managed entities
-USER_FETCH = 'all_groups,all_insts,firstName'
-USER_FETCH_CANCELLED = 'cancelledDate'
-MANAGED_USER_FETCH = 'firstName'
+USER_FETCH = "all_groups,all_insts,firstName"
+USER_FETCH_CANCELLED = "cancelledDate"
+MANAGED_USER_FETCH = "firstName"
# Properties containing user/group/institution information in search query results
-USER_RESULT_PROPERTY = 'people'
-GROUP_RESULT_PROPERTY = 'groups'
-INST_RESULT_PROPERTY = 'institutions'
+USER_RESULT_PROPERTY = "people"
+GROUP_RESULT_PROPERTY = "groups"
+INST_RESULT_PROPERTY = "institutions"
# User and group information we need to populate the Google user directory.
UserEntry = collections.namedtuple(
- 'UserEntry', 'uid cn sn displayName givenName groupIDs groupNames instIDs licensed'
+ "UserEntry", "uid cn sn displayName givenName groupIDs groupNames instIDs licensed"
)
-GroupEntry = collections.namedtuple('GroupEntry', 'groupID groupName description uids')
+GroupEntry = collections.namedtuple("GroupEntry", "groupID groupName description uids")
class LookupRetriever(ConfigurationStateConsumer):
- required_config = ('lookup', 'api_gateway_auth', )
+ required_config = (
+ "lookup",
+ "api_gateway_auth",
+ )
default_api_params = dict(_request_timeout=120)
def __init__(self, *args, **kwargs):
@@ -63,15 +64,15 @@ class LookupRetriever(ConfigurationStateConsumer):
# instance. If a user is in GSuite and is *not* present in this list then they are
# suspended.
eligible_uids = self.get_eligible_uids()
- LOG.info('Total Lookup user entries: %s', len(eligible_uids))
+ LOG.info("Total Lookup user entries: %s", len(eligible_uids))
# Sanity check: there are some eligible users (else Lookup failure?)
if len(eligible_uids) == 0:
- raise RuntimeError('Sanity check failed: no users in eligible set')
+ raise RuntimeError("Sanity check failed: no users in eligible set")
# Get a list of managed users. These are all the people who match the "managed_user_filter"
# in the Lookup settings.
- LOG.info('Reading managed user entries from Lookup')
+ LOG.info("Reading managed user entries from Lookup")
managed_user_entries = self.get_managed_user_entries()
# Form a mapping from uid to managed user.
@@ -79,45 +80,61 @@ class LookupRetriever(ConfigurationStateConsumer):
# Form a set of all *managed user* uids
managed_user_uids = set(managed_user_entries_by_uid.keys())
- LOG.info('Total managed user entries: %s', len(managed_user_uids))
+ LOG.info("Total managed user entries: %s", len(managed_user_uids))
# Sanity check: the managed users should be a subset of the eligible ones.
if len(managed_user_uids - eligible_uids) != 0:
raise RuntimeError(
- 'Sanity check failed: some managed uids were not in the eligible set'
+ "Sanity check failed: some managed uids were not in the eligible set"
)
# Form a set of licensed user uids (anyone with a non-empty misAffiliation)
licensed_uids = {u.uid for u in managed_user_entries if u.licensed}
- LOG.info('Total licensed user entries: %s', len(licensed_uids))
-
- self.state.update({
- 'eligible_uids': eligible_uids,
- 'managed_user_entries_by_uid': managed_user_entries_by_uid,
- 'managed_user_uids': managed_user_uids,
- 'licensed_uids': licensed_uids,
- })
+ LOG.info("Total licensed user entries: %s", len(licensed_uids))
+
+ self.state.update(
+ {
+ "eligible_uids": eligible_uids,
+ "managed_user_entries_by_uid": managed_user_entries_by_uid,
+ "managed_user_uids": managed_user_uids,
+ "licensed_uids": licensed_uids,
+ }
+ )
self.has_retrieved_users = True
def retrieve_cancelled_user_dates(self):
"""
Retrieve information about cancelled dates for suspended google users and form a set of
- those cancelled more than 6 months ago.
+ those cancelled more than 6 months ago and 24 months ago.
"""
+
six_months_ago = date_months_ago(6)
cancelled_six_months_ago_uids = {
- uid for uid, cancelledDate in self.cancelled_dates_by_uid.items()
+ uid
+ for uid, cancelledDate in self.cancelled_dates_by_uid.items()
if cancelledDate and isodate_parse(cancelledDate) < six_months_ago
}
+ LOG.info(
+ "Total Lookup users cancelled more than 6 months ago: %s",
+ len(cancelled_six_months_ago_uids),
+ )
+ self.state.update({"cancelled_six_months_ago_uids": cancelled_six_months_ago_uids})
- LOG.info('Total Lookup users cancelled more than 6 months ago: %s',
- len(cancelled_six_months_ago_uids))
-
- self.state.update({
- 'cancelled_six_months_ago_uids': cancelled_six_months_ago_uids
- })
+ twenty_four_months_ago = date_months_ago(24)
+ cancelled_twenty_four_months_ago_uids = {
+ uid
+ for uid, cancelledDate in self.cancelled_dates_by_uid.items()
+ if cancelledDate and isodate_parse(cancelledDate) < twenty_four_months_ago
+ }
+ LOG.info(
+ "Total Lookup users cancelled more than 24 months ago: %s",
+ len(cancelled_twenty_four_months_ago_uids),
+ )
+ self.state.update(
+ {"cancelled_twenty_four_months_ago_uids": cancelled_twenty_four_months_ago_uids}
+ )
def retrieve_groups(self):
"""
@@ -129,25 +146,25 @@ class LookupRetriever(ConfigurationStateConsumer):
# Get a set containing all groupIDs. These are all the groups that are eligible to be in
# our GSuite instance. If a group is in GSuite and is *not* present in this list then it
# is deleted.
- LOG.info('Reading eligible group entries from Lookup')
+ LOG.info("Reading eligible group entries from Lookup")
eligible_groupIDs = self.get_eligible_groupIDs()
- LOG.info('Total Lookup group entries: %s', len(eligible_groupIDs))
+ LOG.info("Total Lookup group entries: %s", len(eligible_groupIDs))
# Get a set containing all instIDs. These are all the institutions that are eligible to be
# in our GSuite instance. If an institution is in GSuite and is *not* present in this list
# then the corresponding group is deleted.
- LOG.info('Reading eligible institution entries from Lookup')
+ LOG.info("Reading eligible institution entries from Lookup")
eligible_instIDs = self.get_eligible_instIDs()
- LOG.info('Total Lookup institution entries: %s', len(eligible_instIDs))
+ LOG.info("Total Lookup institution entries: %s", len(eligible_instIDs))
# Add these sets together to form the set of all gids (the IDs of all eligible groups and
# institutions).
eligible_gids = eligible_groupIDs | eligible_instIDs
- LOG.info('Total combined Lookup group and institution entries: %s', len(eligible_gids))
+ LOG.info("Total combined Lookup group and institution entries: %s", len(eligible_gids))
# Get a list of managed groups. These are all the groups that match the
# "managed_group_filter" in the Lookup settings.
- LOG.info('Reading managed group entries from Lookup')
+ LOG.info("Reading managed group entries from Lookup")
managed_group_entries = self.get_managed_group_entries()
# Form a mapping from groupID to managed group.
@@ -155,15 +172,14 @@ class LookupRetriever(ConfigurationStateConsumer):
# Form a set of all *managed group* groupIDs
managed_group_groupIDs = set(managed_group_entries_by_groupID.keys())
- LOG.info('Total managed group entries: %s', len(managed_group_groupIDs))
+ LOG.info("Total managed group entries: %s", len(managed_group_groupIDs))
LOG.info(
- 'Total managed group members: %s',
- sum([len(g.uids) for g in managed_group_entries])
+ "Total managed group members: %s", sum([len(g.uids) for g in managed_group_entries])
)
# Get a list of managed institutions. These are all the institutions that match the
# "managed_inst_filter" in the Lookup settings.
- LOG.info('Reading managed institution entries from Lookup')
+ LOG.info("Reading managed institution entries from Lookup")
managed_inst_entries = self.get_managed_inst_entries()
# Form a mapping from instID to managed institution.
@@ -171,72 +187,75 @@ class LookupRetriever(ConfigurationStateConsumer):
# Form a set of all *managed institution* instIDs
managed_inst_instIDs = set(managed_inst_entries_by_instID.keys())
- LOG.info('Total managed institution entries: %s', len(managed_inst_instIDs))
+ LOG.info("Total managed institution entries: %s", len(managed_inst_instIDs))
LOG.info(
- 'Total managed institution members: %s',
- sum([len(i.uids) for i in managed_inst_entries])
+ "Total managed institution members: %s",
+ sum([len(i.uids) for i in managed_inst_entries]),
)
# Add the collections of managed institutions to the collections of managed groups.
managed_group_entries += managed_inst_entries
managed_group_entries_by_gid = {
- **managed_group_entries_by_groupID, **managed_inst_entries_by_instID
+ **managed_group_entries_by_groupID,
+ **managed_inst_entries_by_instID,
}
managed_group_gids = managed_group_groupIDs | eligible_instIDs
LOG.info(
- 'Total combined managed group and institution entries: %s', len(managed_group_gids)
+ "Total combined managed group and institution entries: %s", len(managed_group_gids)
)
LOG.info(
- 'Total combined managed group and institution members: %s',
- sum([len(g.uids) for g in managed_group_entries])
+ "Total combined managed group and institution members: %s",
+ sum([len(g.uids) for g in managed_group_entries]),
)
# Sanity check: the managed groups should be a subset of the eligible ones.
if len(managed_group_gids - eligible_gids) != 0:
raise RuntimeError(
- 'Sanity check failed: some managed gids were not in the eligible set'
+ "Sanity check failed: some managed gids were not in the eligible set"
)
- self.state.update({
- 'eligible_gids': eligible_gids,
- 'managed_group_entries_by_gid': managed_group_entries_by_gid,
- })
+ self.state.update(
+ {
+ "eligible_gids": eligible_gids,
+ "managed_group_entries_by_gid": managed_group_entries_by_gid,
+ }
+ )
###
# Functions to perform Lookup API calls
###
@functools.cached_property
- @cache_to_disk('lookup_eligible_users')
+ @cache_to_disk("lookup_eligible_users")
def eligible_users_by_uid(self):
"""
Dictionary mapping CRSid to UserEntry instances. An entry exists in the dictionary for each
person who is eligible to have a Google account.
"""
- LOG.info('Reading eligible user entries from Lookup')
+ LOG.info("Reading eligible user entries from Lookup")
return {
- person.identifier.value:
- UserEntry(
+ person.identifier.value: UserEntry(
uid=person.identifier.value,
- cn=person.get('registered_name', ''),
- sn=person.get('surname', ''),
- displayName=person.get('display_name', ''),
- givenName=_extract_attribute(person, 'firstName'),
+ cn=person.get("registered_name", ""),
+ sn=person.get("surname", ""),
+ displayName=person.get("display_name", ""),
+ givenName=_extract_attribute(person, "firstName"),
groupIDs={group.groupid for group in person.groups},
groupNames={group.name for group in person.groups},
instIDs={institution.instid for institution in person.institutions},
- licensed=len(person.get('mis_affiliation', '')) > 0,
+ licensed=len(person.get("mis_affiliation", "")) > 0,
)
for person in self._fetch_all_list_results(
self.person_api_client.person_search,
USER_RESULT_PROPERTY,
self.lookup_config.eligible_user_filter,
- extra_props=dict(fetch=USER_FETCH)
- ) if person.identifier.scheme == UID_SCHEME and len(person.identifier.value) > 0
+ extra_props=dict(fetch=USER_FETCH),
+ )
+ if person.identifier.scheme == UID_SCHEME and len(person.identifier.value) > 0
}
@functools.cached_property
- @cache_to_disk('lookup_cancelled_dates')
+ @cache_to_disk("lookup_cancelled_dates")
def cancelled_dates_by_uid(self):
"""
Return a dictionary mapping CRSid to cancelledDate for cancelled users. Limit to
@@ -245,25 +264,28 @@ class LookupRetriever(ConfigurationStateConsumer):
"""
- LOG.info('Reading cancelled date for suspended Google users from Lookup')
+ LOG.info("Reading cancelled date for suspended Google users from Lookup")
uids_to_retrieve = list(self.state.never_logged_in_six_months_uids)
chunk = 100
crsid_chunks = [
- uids_to_retrieve[x:x + chunk]
- for x in range(0, len(uids_to_retrieve), chunk)
+ uids_to_retrieve[x : x + chunk] for x in range(0, len(uids_to_retrieve), chunk)
]
cancelled_users_to_cancelled_date = {}
for crsid_chunk in crsid_chunks:
cancelled_users_to_cancelled_date.update(
{
- person.identifier.value: _extract_attribute(person, 'cancelledDate')
+ person.identifier.value: _extract_attribute(person, "cancelledDate")
for person in self.person_api_client.person_list_people(
- ','.join(crsid_chunk), fetch=USER_FETCH_CANCELLED,
+ ",".join(crsid_chunk),
+ fetch=USER_FETCH_CANCELLED,
**self.default_api_params,
- ).get('result', {}).get('people', [])
- if person.identifier.scheme == UID_SCHEME and len(person.identifier.value) > 0
+ )
+ .get("result", {})
+ .get("people", [])
+ if person.identifier.scheme == UID_SCHEME
+ and len(person.identifier.value) > 0
and person.cancelled
}
)
@@ -271,7 +293,7 @@ class LookupRetriever(ConfigurationStateConsumer):
return cancelled_users_to_cancelled_date
@functools.cached_property
- @cache_to_disk('lookup_eligible_groups')
+ @cache_to_disk("lookup_eligible_groups")
def eligible_groups_by_groupID(self):
"""
Dictionary mapping groupID to GroupEntry instances. An entry exists in the dictionary for
@@ -282,18 +304,18 @@ class LookupRetriever(ConfigurationStateConsumer):
"""
groups = {
- group.groupid:
- GroupEntry(
+ group.groupid: GroupEntry(
groupID=group.groupid,
- groupName=group.get('name', ''),
- description=group.get('description', ''),
- uids=set()
+ groupName=group.get("name", ""),
+ description=group.get("description", ""),
+ uids=set(),
)
for group in self._fetch_all_list_results(
self.group_api_client.group_search,
GROUP_RESULT_PROPERTY,
- self.lookup_config.eligible_group_filter
- ) if len(group.groupid) > 0
+ self.lookup_config.eligible_group_filter,
+ )
+ if len(group.groupid) > 0
}
for crsid, person in self.eligible_users_by_uid.items():
for groupID in person.groupIDs:
@@ -302,7 +324,7 @@ class LookupRetriever(ConfigurationStateConsumer):
return groups
@functools.cached_property
- @cache_to_disk('lookup_eligible_insts')
+ @cache_to_disk("lookup_eligible_insts")
def eligible_insts_by_instID(self):
"""
Dictionary mapping instID to GroupEntry instances. An entry exists in the dictionary for
@@ -320,18 +342,18 @@ class LookupRetriever(ConfigurationStateConsumer):
"""
insts = {
- inst.instid:
- GroupEntry(
+ inst.instid: GroupEntry(
groupID=inst.instid,
- groupName=inst.get('name', ''),
- description=inst.get('name', ''),
- uids=set()
+ groupName=inst.get("name", ""),
+ description=inst.get("name", ""),
+ uids=set(),
)
for inst in self._fetch_all_list_results(
self.inst_api_client.institution_search,
INST_RESULT_PROPERTY,
- self.lookup_config.eligible_inst_filter
- ) if len(inst.instid) > 0
+ self.lookup_config.eligible_inst_filter,
+ )
+ if len(inst.instid) > 0
}
for crsid, person in self.eligible_users_by_uid.items():
for instID in person.instIDs:
@@ -367,7 +389,8 @@ class LookupRetriever(ConfigurationStateConsumer):
"""
return [
- person for _, person in self.eligible_users_by_uid.items()
+ person
+ for _, person in self.eligible_users_by_uid.items()
if self.lookup_config.managed_user_filter is None
or eval(self.lookup_config.managed_user_filter, {}, person._asdict())
]
@@ -378,7 +401,8 @@ class LookupRetriever(ConfigurationStateConsumer):
"""
return [
- group for _, group in self.eligible_groups_by_groupID.items()
+ group
+ for _, group in self.eligible_groups_by_groupID.items()
if self.lookup_config.managed_group_filter is None
or eval(self.lookup_config.managed_group_filter, {}, group._asdict())
]
@@ -389,7 +413,8 @@ class LookupRetriever(ConfigurationStateConsumer):
"""
return [
- inst for _, inst in self.eligible_insts_by_instID.items()
+ inst
+ for _, inst in self.eligible_insts_by_instID.items()
if self.lookup_config.managed_inst_filter is None
or eval(self.lookup_config.managed_inst_filter, {}, inst._asdict())
]
@@ -405,12 +430,11 @@ class LookupRetriever(ConfigurationStateConsumer):
offset = 0
result = []
while offset <= 0 or len(result) >= limit:
- LOG.info(f'Fetching from Lookup API {result_prop}, offset {offset}')
+ LOG.info(f"Fetching from Lookup API {result_prop}, offset {offset}")
response = api_func(
- query=query, **self.default_api_params, **extra_props, limit=limit,
- offset=offset
+ query=query, **self.default_api_params, **extra_props, limit=limit, offset=offset
)
- result = response.get('result', {}).get(result_prop, [])
+ result = response.get("result", {}).get(result_prop, [])
for entity in result:
yield entity
offset += limit
@@ -426,14 +450,10 @@ class LookupRetriever(ConfigurationStateConsumer):
settings = yaml.safe_load(stream)
config = LookupClientConfiguration(
- settings['client_id'],
- settings['client_secret'],
- base_url=settings['base_url']
+ settings["client_id"], settings["client_secret"], base_url=settings["base_url"]
)
return LookupApiClient(config, pool_threads=10)
def _extract_attribute(entity, attr):
- return next(
- (x.value for x in entity.attributes if x.scheme == attr), ''
- )
+ return next((x.value for x in entity.attributes if x.scheme == attr), "")
diff --git a/gsuitesync/sync/main.py b/gsuitesync/sync/main.py
index 64b3f63..1f0e17a 100644
--- a/gsuitesync/sync/main.py
+++ b/gsuitesync/sync/main.py
@@ -14,13 +14,22 @@ from .update import GAPIUpdater
LOG = logging.getLogger(__name__)
-def sync(configuration, *, read_only=True, timeout=300, group_settings=False, just_users=False,
- licensing=False, cache_dir=None):
+def sync(
+ configuration,
+ *,
+ read_only=True,
+ timeout=300,
+ group_settings=False,
+ just_users=False,
+ licensing=False,
+ cache_dir=None,
+ delete_users=False
+):
"""Perform sync given configuration dictionary."""
if read_only:
- LOG.info('Performing synchronisation in READ ONLY mode.')
+ LOG.info("Performing synchronisation in READ ONLY mode.")
else:
- LOG.info('Performing synchronisation in WRITE mode.')
+ LOG.info("Performing synchronisation in WRITE mode.")
# Parse configuration into Configuration dict of appropriate dataclasses
configuration = config.parse_configuration(configuration)
@@ -51,7 +60,7 @@ def sync(configuration, *, read_only=True, timeout=300, group_settings=False, ju
lookup.retrieve_cancelled_user_dates()
# Compare users and optionally groups between Lookup and Google
- comparator = Comparator(configuration, state)
+ comparator = Comparator(configuration, state, delete_users=delete_users)
comparator.compare_users()
if licensing:
comparator.compare_licensing()
@@ -64,7 +73,7 @@ def sync(configuration, *, read_only=True, timeout=300, group_settings=False, ju
comparator.enforce_limits(just_users, licensing)
# Update Google with necessary updates found doing comparison
- updater = GAPIUpdater(configuration, state, read_only=read_only)
+ updater = GAPIUpdater(configuration, state, read_only=read_only, delete_users=delete_users)
updater.update_users()
if licensing:
updater.update_licensing()
diff --git a/gsuitesync/sync/state.py b/gsuitesync/sync/state.py
index d405086..7c3f39c 100644
--- a/gsuitesync/sync/state.py
+++ b/gsuitesync/sync/state.py
@@ -18,6 +18,7 @@ class SyncState:
managed_user_entries_by_uid: dict = field(default_factory=dict)
managed_user_uids: set = field(default_factory=set)
cancelled_six_months_ago_uids: set = field(default_factory=set)
+ cancelled_twenty_four_months_ago_uids: set = field(default_factory=set)
# licensing
licensed_uids: set = field(default_factory=set)
@@ -31,8 +32,8 @@ class SyncState:
directory_service: Optional[discovery.Resource] = None
groupssettings_service: Optional[discovery.Resource] = None
licensing_service: Optional[discovery.Resource] = None
- groups_domain: str = ''
- insts_domain: str = ''
+ groups_domain: str = ""
+ insts_domain: str = ""
################
# Data retrieved from Google
@@ -46,6 +47,8 @@ class SyncState:
never_logged_in_google_uids: set = field(default_factory=set)
never_logged_in_six_months_uids: set = field(default_factory=set)
no_mydrive_shared_settings_uids: set = field(default_factory=set)
+ permission_none_mydrive_shared_result_uids: set = field(default_factory=set)
+ permission_removed_mydrive_shared_result_uids: set = field(default_factory=set)
all_deleted_google_users_by_uid: dict = field(default_factory=dict)
# licensing
google_licensed_uids: set = field(default_factory=set)
diff --git a/gsuitesync/sync/update.py b/gsuitesync/sync/update.py
index a7628ab..8bbe98b 100644
--- a/gsuitesync/sync/update.py
+++ b/gsuitesync/sync/update.py
@@ -2,53 +2,60 @@
Perform the actual updates in Google (unless in read_only mode)
"""
-import logging
import crypt
+import logging
import secrets
+from ..gapiutil import process_requests
from .base import ConfigurationStateConsumer
+from .gapi import MYDRIVE_SHARED_ACTION_SCAN
from .utils import gid_to_email, uid_to_email
-from ..gapiutil import process_requests
LOG = logging.getLogger(__name__)
class GAPIUpdater(ConfigurationStateConsumer):
- required_config = ('sync', 'gapi_domain', 'licensing')
-
- def __init__(self, configuration, state, read_only=True):
- super(GAPIUpdater, self).__init__(configuration, state)
- self.read_only = read_only
+ required_config = ("sync", "gapi_domain", "licensing")
def update_users(self):
process_requests(
self.state.directory_service,
self.user_api_requests(),
- self.sync_config, self.read_only)
+ self.sync_config,
+ self.read_only,
+ )
def update_licensing(self):
if self.licensing_config.product_id is None:
- LOG.info('Skipping licensing assignment update as no Product Id set')
+ LOG.info("Skipping licensing assignment update as no Product Id set")
return
process_requests(
self.state.licensing_service,
self.licensing_api_requests(),
- self.sync_config, self.read_only)
+ self.sync_config,
+ self.read_only,
+ )
def update_groups(self):
process_requests(
self.state.directory_service,
self.group_api_requests(),
- self.sync_config, self.read_only)
+ self.sync_config,
+ self.read_only,
+ )
# Still need to do this even if `not group_settings` as new groups need their settings
process_requests(
self.state.groupssettings_service,
self.group_settings_api_requests(),
- self.sync_config, self.read_only)
+ self.sync_config,
+ self.read_only,
+ )
process_requests(
self.state.directory_service,
self.member_api_requests(),
- self.sync_config, self.read_only)
+ self.sync_config,
+ self.read_only,
+ )
def user_api_requests(self):
"""
@@ -61,13 +68,13 @@ class GAPIUpdater(ConfigurationStateConsumer):
uid: self.state.google_user_updates[uid] for uid in self.state.uids_to_update
}
for uid, update in user_updates.items():
- google_id = self.state.all_google_users_by_uid[uid]['id']
+ google_id = self.state.all_google_users_by_uid[uid]["id"]
# Only show the previous parts of name that have been changed
- updated_google_user_name = update.get('name', {})
- previous_google_user_name = self.state.all_google_users_by_uid[uid].get('name', {})
+ updated_google_user_name = update.get("name", {})
+ previous_google_user_name = self.state.all_google_users_by_uid[uid].get("name", {})
previous = {
- k: previous_google_user_name.get(k, '')
- for k in ['givenName', 'familyName']
+ k: previous_google_user_name.get(k, "")
+ for k in ["givenName", "familyName"]
if k in updated_google_user_name
}
LOG.info('Update user "%s": "%r" from "%r"', uid, update, previous)
@@ -75,68 +82,72 @@ class GAPIUpdater(ConfigurationStateConsumer):
# Suspend old users
for uid in self.state.uids_to_suspend:
- google_id = self.state.all_google_users_by_uid[uid]['id']
+ google_id = self.state.all_google_users_by_uid[uid]["id"]
LOG.info('Suspending user: "%s"', uid)
yield self.state.directory_service.users().update(
- userKey=google_id, body={'suspended': True})
+ userKey=google_id, body={"suspended": True}
+ )
- # Delete suspended users who never logged in
- for uid in self.state.uids_to_delete:
- google_id = self.state.all_google_users_by_uid[uid]['id']
- LOG.info('Deleting user: "%s"', uid)
- yield self.state.directory_service.users().delete(
- userKey=google_id)
+ # Delete suspended users
+ if self.delete_users:
+ for uid in self.state.uids_to_delete:
+ google_id = self.state.all_google_users_by_uid[uid]["id"]
+ LOG.info('Deleting user: "%s"', uid)
+ yield self.state.directory_service.users().delete(userKey=google_id)
# Reactivate returning users
reactivate_body = {
- 'suspended': False,
- 'customSchemas': {
- 'UCam': {
- 'mydrive-shared-action': None,
- 'mydrive-shared-result': None,
- 'mydrive-shared-filecount': None,
+ "suspended": False,
+ "customSchemas": {
+ "UCam": {
+ "mydrive-shared-action": None,
+ "mydrive-shared-result": None,
+ "mydrive-shared-filecount": None,
},
},
}
for uid in self.state.uids_to_reactivate:
- google_id = self.state.all_google_users_by_uid[uid]['id']
+ google_id = self.state.all_google_users_by_uid[uid]["id"]
LOG.info('Reactivating user: "%s"', uid)
yield self.state.directory_service.users().update(
- userKey=google_id, body=reactivate_body)
+ userKey=google_id, body=reactivate_body
+ )
# Restore deleted users
for uid in self.state.uids_to_restore:
- google_id = self.state.all_deleted_google_users_by_uid[uid]['id']
+ google_id = self.state.all_deleted_google_users_by_uid[uid]["id"]
LOG.info('Restoring user: "%s" (%s)', uid, google_id)
yield self.state.directory_service.users().undelete(
- userKey=google_id,
- body={'orgUnitPath': self.sync_config.new_user_org_unit_path})
+ userKey=google_id, body={"orgUnitPath": self.sync_config.new_user_org_unit_path}
+ )
# Create new users
for uid in self.state.uids_to_add:
# Generate a random password which is thrown away.
- new_user = {**{
- 'hashFunction': 'crypt',
- 'password': crypt.crypt(secrets.token_urlsafe(), crypt.METHOD_SHA512),
- 'orgUnitPath': self.sync_config.new_user_org_unit_path,
- }, **self.state.google_user_creations[uid]}
- redacted_user = {**new_user, **{'password': 'REDACTED'}}
+ new_user = {
+ **{
+ "hashFunction": "crypt",
+ "password": crypt.crypt(secrets.token_urlsafe(), crypt.METHOD_SHA512),
+ "orgUnitPath": self.sync_config.new_user_org_unit_path,
+ },
+ **self.state.google_user_creations[uid],
+ }
+ redacted_user = {**new_user, **{"password": "REDACTED"}}
LOG.info('Adding user "%s": %s', uid, redacted_user)
yield self.state.directory_service.users().insert(body=new_user)
# Mark users to have their MyDrive scanned
scan_body = {
- 'customSchemas': {
- 'UCam': {
- 'mydrive-shared-action': 'scan',
+ "customSchemas": {
+ "UCam": {
+ "mydrive-shared-action": MYDRIVE_SHARED_ACTION_SCAN,
},
},
}
for uid in self.state.uids_to_scan_mydrive:
- google_id = self.state.all_google_users_by_uid[uid]['id']
+ google_id = self.state.all_google_users_by_uid[uid]["id"]
LOG.info('Set MyDrive to be scanned for user: "%s"', uid)
- yield self.state.directory_service.users().update(
- userKey=google_id, body=scan_body)
+ yield self.state.directory_service.users().update(userKey=google_id, body=scan_body)
def licensing_api_requests(self):
"""
@@ -153,27 +164,28 @@ class GAPIUpdater(ConfigurationStateConsumer):
# Remove licences
for uid in self.state.licences_to_remove:
- userId = f'{uid}@{self.gapi_domain_config.name}'
+ userId = f"{uid}@{self.gapi_domain_config.name}"
skuId = self.state.google_skus_by_uid.get(uid)
LOG.info('Removing licence from user: "%s" (%s)', uid, skuId)
yield self.state.licensing_service.licenseAssignments().delete(
- userId=userId, skuId=skuId, productId=self.licensing_config.product_id)
+ userId=userId, skuId=skuId, productId=self.licensing_config.product_id
+ )
available_by_sku[skuId] += 1
# Add licences
for index, uid in enumerate(self.state.licences_to_add):
- userId = f'{uid}@{self.gapi_domain_config.name}'
+ userId = f"{uid}@{self.gapi_domain_config.name}"
skuId = next_available_sku()
if skuId is None:
LOG.warning(
- 'No more available licences to allocate missing licenses for %d user(s)',
- len(self.state.licences_to_add - index)
+ "No more available licences to allocate missing licenses for %d user(s)",
+ len(self.state.licences_to_add - index),
)
return
LOG.info('Adding licence to user: "%s" (%s)', uid, skuId)
yield self.state.licensing_service.licenseAssignments().insert(
- skuId=skuId, productId=self.licensing_config.product_id,
- body={'userId': userId})
+ skuId=skuId, productId=self.licensing_config.product_id, body={"userId": userId}
+ )
available_by_sku[skuId] -= 1
def group_api_requests(self):
@@ -187,13 +199,13 @@ class GAPIUpdater(ConfigurationStateConsumer):
gid: self.state.google_group_updates[gid] for gid in self.state.gids_to_update
}
for gid, update in group_updates.items():
- google_id = self.state.all_google_groups_by_gid[gid]['id']
+ google_id = self.state.all_google_groups_by_gid[gid]["id"]
LOG.info('Update group "%s": "%r"', gid, update)
yield self.state.directory_service.groups().patch(groupKey=google_id, body=update)
# Delete cancelled groups
for gid in self.state.gids_to_delete:
- google_id = self.state.all_google_groups_by_gid[gid]['id']
+ google_id = self.state.all_google_groups_by_gid[gid]["id"]
LOG.info('Deleting group: "%s"', gid)
yield self.state.directory_service.groups().delete(groupKey=google_id)
@@ -215,7 +227,8 @@ class GAPIUpdater(ConfigurationStateConsumer):
user_key = uid_to_email(uid, self.gapi_domain_config.name)
LOG.info('Adding user "%s" to group "%s"', user_key, group_key)
yield self.state.directory_service.members().insert(
- groupKey=group_key, body={'email': user_key})
+ groupKey=group_key, body={"email": user_key}
+ )
# Delete removed members
for gid, uid in self.state.members_to_delete:
@@ -223,7 +236,8 @@ class GAPIUpdater(ConfigurationStateConsumer):
user_key = uid_to_email(uid, self.gapi_domain_config.name)
LOG.info('Removing user "%s" from group "%s"', user_key, group_key)
yield self.state.directory_service.members().delete(
- groupKey=group_key, memberKey=user_key)
+ groupKey=group_key, memberKey=user_key
+ )
def group_settings_api_requests(self):
"""
@@ -237,7 +251,8 @@ class GAPIUpdater(ConfigurationStateConsumer):
settings = self.sync_config.group_settings
LOG.info('Updating settings for new group "%s": %s', gid, settings)
yield self.state.groupssettings_service.groups().patch(
- groupUniqueId=email, body=settings)
+ groupUniqueId=email, body=settings
+ )
# Update existing group settings (will be empty of `not group_settings`)
for gid in self.state.gids_to_update_group_settings:
@@ -245,4 +260,5 @@ class GAPIUpdater(ConfigurationStateConsumer):
settings = self.state.group_settings_to_update[gid]
LOG.info('Updating settings for existing group "%s": %s', gid, settings)
yield self.state.groupssettings_service.groups().patch(
- groupUniqueId=email, body=settings)
+ groupUniqueId=email, body=settings
+ )
diff --git a/gsuitesync/sync/utils.py b/gsuitesync/sync/utils.py
index af8330b..280e3c5 100644
--- a/gsuitesync/sync/utils.py
+++ b/gsuitesync/sync/utils.py
@@ -25,31 +25,37 @@ LOG = logging.getLogger(__name__)
# that follows, we use "gid" to refer to the unique identifier of the group or institution in
# Lookup (i.e., gid may be either a Lookup groupID or instID).
-user_email_regex = re.compile('^[a-z][a-z0-9]{3,7}@.*$')
-groupID_regex = re.compile('^[0-9]{6,8}$')
-instID_regex = re.compile('^[A-Z][A-Z0-9]+$')
+user_email_regex = re.compile("^[a-z][a-z0-9]{3,7}@.*$")
+groupID_regex = re.compile("^[0-9]{6,8}$")
+instID_regex = re.compile("^[A-Z][A-Z0-9]+$")
def email_to_uid(email):
- return email.split('@')[0] if user_email_regex.match(email) else None
+ return email.split("@")[0] if user_email_regex.match(email) else None
def email_to_gid(email):
- gid = email.split('@')[0]
+ gid = email.split("@")[0]
return (
- gid if groupID_regex.match(gid) else
- gid.upper() if instID_regex.match(gid.upper()) else None
+ gid
+ if groupID_regex.match(gid)
+ else gid.upper()
+ if instID_regex.match(gid.upper())
+ else None
)
def uid_to_email(uid, domain):
- return f'{uid}@{domain}'
+ return f"{uid}@{domain}"
def gid_to_email(gid, groups_domain, insts_domain):
return (
- f'{gid}@{groups_domain}' if groupID_regex.match(gid) else
- f'{gid.lower()}@{insts_domain}' if instID_regex.match(gid) else None
+ f"{gid}@{groups_domain}"
+ if groupID_regex.match(gid)
+ else f"{gid.lower()}@{insts_domain}"
+ if instID_regex.match(gid)
+ else None
)
@@ -64,7 +70,7 @@ def isodate_parse(str):
def custom_action(user, property):
- return user.get('customSchemas', {}).get('UCam', {}).get(property)
+ return user.get("customSchemas", {}).get("UCam", {}).get(property)
def cache_to_disk(file_name):
@@ -76,26 +82,28 @@ def cache_to_disk(file_name):
cache file name. e.g. 'google_groups_{domain}'
"""
- def decorator(method):
+ def decorator(method):
@wraps(method)
def _impl(self, *args, **kwargs):
# Skip if file caching not enabled
if not self.cache_dir:
return method(self, *args, **kwargs)
# Check if cache file exists1
- full_path = path.join(self.cache_dir, file_name.format(**kwargs)) + '.yaml'
+ full_path = path.join(self.cache_dir, file_name.format(**kwargs)) + ".yaml"
if path.exists(full_path):
- LOG.info(f'Reading from cache file: {full_path}')
+ LOG.info(f"Reading from cache file: {full_path}")
# Read and return file result instead
with open(full_path) as fp:
data = yaml.unsafe_load(fp)
return data
# Get the real data and cache it to file
data = method(self, *args, **kwargs)
- LOG.info(f'Writing to cache file: {full_path}')
- with open(full_path, 'w') as fp:
+ LOG.info(f"Writing to cache file: {full_path}")
+ with open(full_path, "w") as fp:
yaml.dump(data, fp)
return data
+
return _impl
+
return decorator
diff --git a/requirements.txt b/requirements.txt
index 07f6055..7902d48 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
-PyYAML
docopt
google-api-python-client
google-auth
-ucam-identitylib==1.0.10
python-dateutil
+PyYAML
+ucam-identitylib==1.0.10
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 9c7b389..2a192c4 100644
--- a/setup.py
+++ b/setup.py
@@ -8,22 +8,21 @@ def load_requirements():
Load requirements file and return non-empty, non-comment lines with leading and trailing
whitespace stripped.
"""
- with open(os.path.join(os.path.dirname(__file__), 'requirements.txt')) as f:
+ with open(os.path.join(os.path.dirname(__file__), "requirements.txt")) as f:
return [
- line.strip() for line in f
- if line.strip() != '' and not line.strip().startswith('#')
+ line.strip() for line in f if line.strip() != "" and not line.strip().startswith("#")
]
setup(
- name='gsuitesync',
- version='0.13.0',
+ name="gsuitesync",
+ version="0.14.0",
packages=find_packages(),
install_requires=load_requirements(),
entry_points={
- 'console_scripts': [
- 'gsuitesync=gsuitesync:main',
+ "console_scripts": [
+ "gsuitesync=gsuitesync:main",
]
},
- python_requires='>=3.9',
+ python_requires=">=3.9",
)
diff --git a/tox.ini b/tox.ini
index 2aa3e15..e6221b9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,7 +9,7 @@
#
[tox]
# Envs which should be run by default.
-envlist=flake8,py3
+envlist=black,flake8,py3
# Allow overriding toxworkdir via environment variable
toxworkdir={env:TOXINI_WORK_DIR:{toxinidir}/.tox}
# Avoid .egg-info directories
@@ -60,4 +60,17 @@ deps=
flake8=={env:TOXINI_FLAKE8_VERSION:3.9.2}
commands=
flake8 --version
- flake8 .
+ # flake8 (pycodestyle) is not PEP8 on E203
+ # https://github.com/PyCQA/pycodestyle/issues/373
+ flake8 --extend-ignore=E203 .
+
+[testenv:black]
+basepython=python3
+deps=
+# We specify a specific version of black to avoid introducing "false"
+# regressions when new checks are introduced. The version of black used may
+# be overridden via the TOXINI_BLACK_VERSION environment variable.
+ black=={env:TOXINI_BLACK_VERSION:23.1.0}
+commands=
+ black --version
+ black --check --diff --line-length 99 gsuitesync setup.py
--
GitLab