""" Utility functions which should have been part of the Google API client. """ import logging from googleapiclient.errors import HttpError from time import sleep LOG = logging.getLogger(__name__) def list_all(list_cb, *, page_size=500, retries=2, retry_delay=5, items_key='items', **kwargs): """ Simple wrapper for Google Client SDK list()-style callables. Repeatedly fetches pages of results merging all the responses together. Returns the merged "items" arrays from the responses. The key used to get the "items" array from the response may be overridden via the items_key argument. """ # Loop while we wait for nextPageToken to be "none" page_token = None resources = [] while True: try: list_response = list_cb(pageToken=page_token, maxResults=page_size, **kwargs).execute() except HttpError as err: if (err.resp.status == 503 and retries > 0): retries -= 1 LOG.warn('503: Service unavailable - retrying') sleep(retry_delay) continue if retries == 0: LOG.error('503: Service unavailable - retry count exceeded') raise resources.extend(list_response.get(items_key, [])) # Get the token for the next page page_token = list_response.get('nextPageToken') if page_token is None: break return resources