add retries to OAuth2 HTTP(S) requests

Add a configurable number of retry attempts when fetching tokens from
the OAuth2 endpoint or when requesting resources authorised with OAuth2
tokens.

This uses requests' built-in retry behaviour which retries only in the
face of network errors such as DNS failures or connection errors. It
does not retry if data has made it to the server.

It is, unfortunately, non-trivial to write a test for this since faking
a network error requires deeper knowledge of requests internals than I
possess. We rely on the max_retries parameter behaving as documented.

lookupapi/authentication.py

@@ -8,6 +8,7 @@ from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.core.exceptions import ObjectDoesNotExist
 from rest_framework.authentication import BaseAuthentication
+from requests.adapters import HTTPAdapter
 from requests_oauthlib import OAuth2Session
 from oauthlib.oauth2 import BackendApplicationClient, TokenExpiredError
 
@@ -23,6 +24,9 @@ def _get_session():
     """
     client = BackendApplicationClient(client_id=settings.LOOKUP_API_OAUTH2_CLIENT_ID)
     session = OAuth2Session(client=client)
+    adapter = HTTPAdapter(max_retries=settings.LOOKUP_API_OAUTH2_MAX_RETRIES)
+    session.mount('http://', adapter)
+    session.mount('https://', adapter)
     session.fetch_token(
         timeout=2, token_url=settings.LOOKUP_API_OAUTH2_TOKEN_URL,
         client_id=settings.LOOKUP_API_OAUTH2_CLIENT_ID,

lookupapi/defaultsettings.py

@@ -69,3 +69,11 @@ List of OAuth2 scopes the API server will request for the token it will use with
 introspection endpoint.
 
 """
+
+LOOKUP_API_OAUTH2_MAX_RETRIES = 5
+"""
+Maximum number of retries when fetching URLs from the OAuth2 endpoint or OAuth2 authenticated URLs.
+This applies only to failed DNS lookups, socket connections and connection timeouts, never to
+requests where data has made it to the server.
+
+"""