Merge branch 'issue-2-configure-batch' into 'master'

allow configuration of API batch size and inter-batch delay

Closes #2

See merge request !2

configuration-example.yaml

@@ -10,6 +10,15 @@ sync:
   # completely outside of this script.
   ignore_google_org_unit_path_regex: '^/Service Accounts$'
 
+  # Inter-batch delay in seconds. This is useful to avoid hitting Google rate
+  # limits. Default: 5.
+  inter_batch_delay: 5
+
+  # Batch size for Google API calls. Google supports batching requests together
+  # into one API call. This can be no greater than 1000 but in practice this
+  # should be less to avoid hitting other Google rate limits. Default: 50.
+  batch_size: 50
+
 # Configure limits defining maximum scope of changes.
 limits:
   # The abort_... settings below are safety limits and will abort the run if the

gsuitesync/sync.py

@@ -6,8 +6,10 @@ import crypt
 import dataclasses
 import itertools
 import logging
+import numbers
 import re
 import secrets
+import time
 import typing
 
 from googleapiclient import discovery
@@ -42,6 +44,13 @@ class Configuration(config.ConfigurationDataclassMixin):
     # are managed completely outside of this script.
     ignore_google_org_unit_path_regex: typing.Union[str, None] = None
 
+    # Inter-batch delay in seconds. This is useful to avoid hitting Google rate limits.
+    inter_batch_delay: numbers.Real = 5
+
+    # Batch size for Google API calls. Google supports batching requests together into one API
+    # call.
+    batch_size: int = 50
+
 
 def sync(configuration, *, read_only=True):
     """Perform sync given configuration dictionary."""
@@ -277,7 +286,7 @@ def sync(configuration, *, read_only=True):
     # Make an chunked iterator of requests to the directory API. The Directory API supports a
     # maximum batch size of 1000. See:
     # https://developers.google.com/admin-sdk/directory/v1/guides/batch
-    for request_batch in _grouper(api_requests(), n=1000):
+    for request_batch in _grouper(api_requests(), n=sync_config.batch_size):
         # Form batch request.
         batch = directory_service.new_batch_http_request()
         for request in request_batch:
@@ -286,6 +295,7 @@ def sync(configuration, *, read_only=True):
         # Execute the batch request if not in read only mode. Otherwise log that we would have.
         if not read_only:
             LOG.info('Issuing batch request to Google.')
+            time.sleep(sync_config.inter_batch_delay)
             batch.execute()
         else:
             LOG.info('Not issuing batch request in read-only mode.')