gapiutil.py

"""
Utility functions which should have been part of the Google API client.

"""
import logging
from googleapiclient.errors import HttpError
from time import sleep

LOG = logging.getLogger(__name__)


def list_all(list_cb, *, page_size=500, retries=2, retry_delay=5, items_key='items',  **kwargs):
    """
    Simple wrapper for Google Client SDK list()-style callables. Repeatedly fetches pages of
    results merging all the responses together. Returns the merged "items" arrays from the
    responses. The key used to get the "items" array from the response may be overridden via the
    items_key argument.

    """
    # Loop while we wait for nextPageToken to be "none"
    page_token = None
    resources = []
    while True:
        try:
            list_response = list_cb(pageToken=page_token, maxResults=page_size, **kwargs).execute()
        except HttpError as err:
            if (err.resp.status == 503 and retries > 0):
                retries -= 1
                LOG.warn('503: Service unavailable - retrying')
                sleep(retry_delay)
                continue
            if retries == 0:
                LOG.error('503: Service unavailable - retry count exceeded')
            raise
        resources.extend(list_response.get(items_key, []))

        # Get the token for the next page
        page_token = list_response.get('nextPageToken')
        if page_token is None:
            break

    return resources