merging dev changes - 26-11-24

.gitlab-ci.yml

@@ -47,13 +47,13 @@ stages:
       - python3 -V  # Print out python version for debugging
       - mkdir -p $PACKAGES_DIR
       - mkdir -p $OUTPUT_DIR
-      - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-source-generation.git" $PACKAGES_DIR/source_gen
-      - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_plotting.git" $PACKAGES_DIR/plotting
-      - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-epimodel.git" $PACKAGES_DIR/epimodel
-      - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-advisory-builder.git" $PACKAGES_DIR/advisory_builder
-      - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews_environmental_suitability_v2.git" $PACKAGES_DIR/environmental_suitability
-      - git clone -b $BRANCH "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing
-      - git clone -b main "https://nouser:$GROUP_WHEAT_ACCESS_VAR@gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-met-processing.git" $PACKAGES_DIR/met_processing
+      - git clone -b $BRANCH "https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.developers.cam.ac.uk/gilligan-epid/dewas-wheat-rusts/ews-source-calculation.git" $PACKAGES_DIR/source_gen
+      - git clone -b $BRANCH "https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.developers.cam.ac.uk/gilligan-epid/dewas-wheat-rusts/ews-plotting.git" $PACKAGES_DIR/plotting
+      - git clone -b $BRANCH "https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.developers.cam.ac.uk/gilligan-epid/dewas-wheat-rusts/ews-epidemiology.git" $PACKAGES_DIR/epimodel
+      - git clone -b main "https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.developers.cam.ac.uk/gilligan-epid/dewas-wheat-rusts/ews-advisory-builder.git" $PACKAGES_DIR/advisory_builder
+      - git clone -b main "https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.developers.cam.ac.uk/gilligan-epid/dewas-wheat-rusts/ews-environmental-suitability.git" $PACKAGES_DIR/environmental_suitability
+      - git clone -b $BRANCH "https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.developers.cam.ac.uk/gilligan-epid/dewas-wheat-rusts/ews-postprocessing.git" $PACKAGES_DIR/post_processing
+      - git clone -b main "https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.developers.cam.ac.uk/gilligan-epid/dewas-wheat-rusts/ews-met-processing.git" $PACKAGES_DIR/met_processing
       - epimodel=$PACKAGES_DIR/epimodel
       - advisory=$PACKAGES_DIR/advisory_builder
       - met_processing=$PACKAGES_DIR/met_processing
@@ -355,11 +355,6 @@ pages:
     - mkdir $CI_PROJECT_DIR/public
     - mv -v $CI_PROJECT_DIR/docs/_build/html/* $CI_PROJECT_DIR/public/
     - ls $CI_PROJECT_DIR/public/
-  variables:
-#    PAGES_PREFIX: ""
-    PAGES_PREFIX: "latest-release"
-  pages:
-    path_prefix: "$PAGES_PREFIX"
   artifacts:
     paths:
       - public

docs/_source/installation.rst

@@ -57,27 +57,43 @@ chmod -R g+sw <install_location>
 
 make all files written here default to the current user group, and group-writable. This is not necessary if you are depoying in a single-user environment, such as a personal computer
 
-2: Download code
+2: Clone code
 -------------
-Download the latest source code from the releases page on GitLab.
+(can copy-paste below as commands to run on one line)
 
-https://gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-east-africa-configs/-/releases
+cd into <install_location>/code
+
+.. code-block:: bash
 
-cd into <install_location>/code and unzip the code.
+   git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-coordinator.git coordinator;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-environmental-suitability.git environmental_suitability;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-advisory-builder.git advisory_builder;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-plotting.git plotting;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-epidemiology.git epimodel;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-source-calculation.git source_gen;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-met-processing.git met_extractor_v2;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-postprocessing.git post_processing;
 
-this should leave you with the following dirs:
+or generate this string in bash:
+
+.. code-block:: bash
 
-advisory-builder
-coordinator
-environmental_suitability
-epidemiology
-met_extractor_v2
-plotting
-post_processing
+   for package in ews-coordinator,coordinator \
+                  ews-environmental-suitability,environmental_suitability \
+                  ews-advisory-builder,advisory_builder \
+                  ews-plotting,plotting \
+                  ews-epidemiology,epimodel \
+                  ews-source-calculation,source_gen \
+                  ews-met-processing,met_extractor_v2 \
+                  ews-postprocessing,post_processing;
+   do
+     repo=`echo $package | cut -d "," -f 1`
+     folder=`echo $package | cut -d "," -f 2`
+     printf "%s" "git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/${repo}.git ${folder};"
+   done
+
+shell scripts to perform the above operations:
+:download:`Download .cmd file <../_static/assets/windows_clone_git.cmd>`
+:download:`Download .sh file <../_static/assets/linux_clone_git.sh>`
 
 3: Copy the assets needed for the various regions:
 ---------------------------------------------------------
-Download the region assets zip from the release page linked above.
+Download the region assets zip from the release page on GitLab.
+
+https://gitlab.developers.cam.ac.uk/gilligan-epid/wheat-rusts/ews-east-africa-configs/-/releases
 
 .. admonition:: Add assets for one region at a time
 
@@ -113,7 +129,7 @@ sys_config_EastAfrica_fc_live.json
 sys_config_SouthAsia_fc_live.json
 
 Edit the newly copied sys_config files and edit the root directories to match your installation. These instructions are
-based on using the pre-prepared Docker image, which should use the following values:
+based on using the pre-prepared Docker image, which should use the following values (these paths are used by the docker image):
 
 .. code-block:: bash
 
@@ -265,8 +281,8 @@ Edit config files to match your installation
 
 **Note than when we refer to "the config files" we are referring to the main JSON configs that are passed in from the command
 line when running the pipeline.
-**each region and component will have its own config file, e.g. depo_config_EastAfrica_fc_live.json, depo_config_SouthAsia_fc_live.json.**
-(e.g.** ``/storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json``
+each region and component will have its own config file, e.g. depo_config_EastAfrica_fc_live.json, depo_config_SouthAsia_fc_live.json.**
+(e.g. ``/storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/sys_config_EastAfrica_fc_live.json``
 ``/storage/app/EWS_prod/regions/EastAfrica/resources/configs/coordinator/fc/depo_config_EastAfrica_fc_live.json``).
 
 **Files will be copied using the "scp" command, so your computer will need to be able to run this command if you are not

docs/api.rst

@@ -6,7 +6,7 @@ API
    :template: custom-module-template.rst
    :recursive:
 
-..    ews.coordinator
+    ews.coordinator
     ews.plotting
     ews.postprocessing
     ews.met_processing

ews/coordinator/processor_deposition.py

 #ProcessorDeposition.py
 '''Functions to process the deposition component.'''
 
-import logging
+import  logging
 import os
 import datetime
 from argparse import ArgumentParser

ews/coordinator/utils/buffering_smtp_handler.py

-import json
-import logging
-import logging.handlers
-import smtplib
-from typing import Tuple, List
-
-
-class BufferingSMTPHandler(logging.handlers.BufferingHandler):
-
-    # BASED ON CODE DEFINED HERE:
-    # https://gist.github.com/anonymous/1379446 LINKED OUT FROM THIS ISSUE:
-    # https://stackoverflow.com/questions/1610845/collate-output-in-python-logging-memoryhandler-with-smtphandler
-
-    def __init__(self,
-                 server: Tuple[str, str],
-                 credentials: Tuple[str, str],
-                 fromaddr: str, toaddrs: List[str], subject: str, capacity: int):
-
-        logging.handlers.BufferingHandler.__init__(self, capacity)
-        self.mailhost = server[0]
-        self.mailport = server[1]
-        self.login = credentials[0]
-        self.passwd = credentials[1]
-        self.fromaddr = fromaddr
-        self.toaddrs = toaddrs
-        self.subject = subject
-        self.setFormatter(logging.Formatter("%(asctime)s %(levelname)-5s %(message)s"))
-
-    def flush(self):
-        if len(self.buffer) > 0:
-            try:
-                port = self.mailport
-                server = smtplib.SMTP(self.mailhost, port)
-                msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n" % (self.fromaddr, ','.join(self.toaddrs), self.subject)
-                for record in self.buffer:
-                    s = self.format(record)
-                    print(s)
-                    msg = msg + s + "\r\n"
-
-                server.ehlo()
-                server.starttls()
-                server.login(self.login, self.passwd)
-                server.sendmail(self.fromaddr, self.toaddrs, msg)
-                server.quit()
-            except:
-                raise Exception("Error thrown in BufferingSMTPHandler")
-            self.buffer = []

ews/coordinator/utils/processor_utils.py

@@ -383,19 +383,6 @@ def setup_logging(job_path: str,
     is_live: bool = config["live"]
     log_level: str = config["log_level"]
 
-    # get the email credentials file path from the environment variables
-    assert 'EMAIL_CRED' in os.environ
-    email_credential_fn = os.environ['EMAIL_CRED']
-    assert os.path.exists(email_credential_fn)
-
-    with open(email_credential_fn, 'r') as f:
-        gmail_config = json.load(f)
-
-    # check contents
-    required_keys = ['user', 'pass', 'host', 'port', 'toaddrs']
-    for required_key in required_keys:
-        assert required_key in gmail_config
-
     arg_string = ' '.join(sys.argv)
 
     # load logging json config file
@@ -416,21 +403,6 @@ def setup_logging(job_path: str,
 
     log_config_dict['handlers']['handler_job']['filename'] = log_file_path
 
-    if is_live:
-        # TODO: smtp handler can only use tls, but ssl is more secure. Look into defining/writing a suitable smtp handler
-        log_config_dict['handlers']['handler_buffered_email']['server'] = (gmail_config['host'], gmail_config['port'])
-        log_config_dict['handlers']['handler_buffered_email']['credentials'] = (gmail_config['user'], gmail_config['pass'])
-        log_config_dict['handlers']['handler_buffered_email']['fromaddr'] = gmail_config['user']
-        log_config_dict['handlers']['handler_buffered_email']['toaddrs'] = gmail_config['toaddrs']
-    else:
-        # if not live - we dont want to send emails
-        print(f"is_live = False, so removing email handler from logging config")
-        log_config_dict['handlers'].pop('handler_buffered_email')
-        for logger_key in log_config_dict['loggers'].keys():
-            handlers = log_config_dict['loggers'][logger_key]['handlers']
-            if 'handler_buffered_email' in handlers:
-                handlers.remove('handler_buffered_email')
-
     assert log_level in loglevels
     for logger_key in log_config_dict['loggers'].keys():
         log_config_dict['loggers'][logger_key]['level'] = loglevels.get(log_level)

tests/integration/partial/integration_test_utils.py

@@ -63,14 +63,9 @@ class IntegrationTestUtils:
     @staticmethod
     def check_resources_exist():
         """
-        do a single check for the email cred file, which will raise an error suggesting that the user may not have set
-        their pythonpath correctly
+        currently redundant, keeping this stub for future use
         """
-        emailcred_exists = os.path.exists(IntegrationTestUtils.EMAIL_CRED_PATH)
-        if not emailcred_exists:
-            raise FileNotFoundError(f"email cred file {IntegrationTestUtils.EMAIL_CRED_PATH} not found - have you set "
-                                    f"the working directory to the same directory as the tests you are running?"
-                                    f" (tests use a path relative to the test directory)")
+        pass
 
     @staticmethod
     def build_arg_parser() -> argparse.ArgumentParser:

tests/test_data/test_deployment/envs/Cred_gmail.json

-{
-    "user" : "",
-    "pass" : "****",
-    "host" : "smtp.gmail.com",
-    "port" : 587,
-    "toaddrs" : [""]
-}
\ No newline at end of file

tests/test_data/test_deployment/envs/test_log_config.json

@@ -32,20 +32,6 @@
       "filters": [
         "mask_passwords"
       ]
-    },
-    "handler_buffered_email": {
-      "class": "ews.coordinator.utils.buffering_smtp_handler.BufferingSMTPHandler",
-      "level": "ERROR",
-      "server": "NOT_SET",
-      "credentials": "NOT_SET",
-      "fromaddr": "NOT_SET",
-      "toaddrs": "NOT_SET",
-      "subject": "ERROR in EWS Processor",
-      "formatter": "detailed",
-      "filters": [
-        "mask_passwords"
-      ],
-      "capacity": 100
     }
   },
   "loggers": {