Retry API requests when a HTTP 503 "Service unavaliable" is received

Closes: #6 (closed)

Also puts in a sanity check for users missing 'orgUnitPath', at least helping to identify the issue with #8 (closed) (if not closing)

Note: failed to get any 503 while testing - probably have to let it out in the wild and wait for pipeline failures

gsuitesync/gapiutil.py

@@ -2,9 +2,14 @@
 Utility functions which should have been part of the Google API client.
 
 """
+import logging
+from googleapiclient.errors import HttpError
+from time import sleep
 
+LOG = logging.getLogger(__name__)
 
-def list_all(list_cb, *, page_size=500, items_key='items',  **kwargs):
+
+def list_all(list_cb, *, page_size=500, retries=2, retry_delay=5, items_key='items',  **kwargs):
     """
     Simple wrapper for Google Client SDK list()-style callables. Repeatedly fetches pages of
     results merging all the responses together. Returns the merged "items" arrays from the
@@ -16,7 +21,17 @@ def list_all(list_cb, *, page_size=500, items_key='items',  **kwargs):
     page_token = None
     resources = []
     while True:
-        list_response = list_cb(pageToken=page_token, maxResults=page_size, **kwargs).execute()
+        try:
+            list_response = list_cb(pageToken=page_token, maxResults=page_size, **kwargs).execute()
+        except HttpError as err:
+            if (err.resp.status == 503 and retries > 0):
+                retries -= 1
+                LOG.warn('503: Service unavailable - retrying')
+                sleep(retry_delay)
+                continue
+            if retries == 0:
+                LOG.error('503: Service unavailable - retry count exceeded')
+            raise
         resources.extend(list_response.get(items_key, []))
 
         # Get the token for the next page