Add retry decorator

Zarr functions to write to disk fail when running many
processes. This allows to retry on failure.

stpt_pipeline/preprocess.py

 import logging
 import re
+import time
+from functools import wraps
 from os import listdir
 from pathlib import Path
 
@@ -21,13 +23,48 @@ log = logging.getLogger('owl.daemon.pipeline')
 blosc.use_threads = False  # TODO: Check if this makes it quicker or slower
 
 
-def read_flatfield(flat_file: Path):
-    """[summary]
+def retry(exceptions, tries=4, delay=3, backoff=2):
+    """Retry calling the decorated function using an exponential backoff.
 
     Parameters
     ----------
-    flat_file : [type]
-        [description]
+    exceptions
+        The exception to check, May be a tuple of exceptions.
+    tries
+        Number of times to try (not retry) before giving up.
+    delay
+        Initial delay between retries in seconds.
+    backoff
+        Backoff multiplier (e.g. value of 2 will double the delay
+        each retry).
+    """
+    def deco_retry(f):
+        @wraps(f)
+        def f_retry(*args, **kwargs):
+            mtries, mdelay = tries, delay
+            while mtries > 1:
+                try:
+                    return f(*args, **kwargs)
+                except exceptions as e:
+                    msg = '{}, Retrying in {} seconds...'.format(e, mdelay)
+                    log.warning(msg)
+                    time.sleep(mdelay)
+                    mtries -= 1
+                    mdelay *= backoff
+            return f(*args, **kwargs)
+
+        return f_retry  # true decorator
+
+    return deco_retry
+
+
+def read_flatfield(flat_file: Path) -> np.ndarray:
+    """Read stored flatfield image.
+
+    Parameters
+    ----------
+    flat_file
+        Full path to flatfield image in npy format.
 
     Returns
     -------