From d8f6227c93070ace1da9021bd24c65fc0421df96 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Wed, 5 Jul 2023 13:54:52 +0100
Subject: [PATCH] more doc work
---
.../build/{Dockerfile_template => Dockerfile} | 0
docs/_source/installation.rst | 187 +++++++++++++++++-
docs/api.rst | 2 +-
docs/index.rst | 2 +
4 files changed, 181 insertions(+), 10 deletions(-)
rename configs/docker/build/{Dockerfile_template => Dockerfile} (100%)
diff --git a/configs/docker/build/Dockerfile_template b/configs/docker/build/Dockerfile
similarity index 100%
rename from configs/docker/build/Dockerfile_template
rename to configs/docker/build/Dockerfile
diff --git a/docs/_source/installation.rst b/docs/_source/installation.rst
index b0d8354..1b262cf 100644
--- a/docs/_source/installation.rst
+++ b/docs/_source/installation.rst
@@ -3,17 +3,186 @@ Installation
.. _installation:
+1: Create dirs:
+---------------
+If you are planning to have a dedicated user/group for this pipeline, make sure you run these commands using that user
-To install the package, do stuff
+create these dirs as user “ewsmanagerâ€, or chmod them afterwards
-for sections
-============
+.. code-block:: bash
-for subsections
----------------
+ cd <install_location>
+ mkdir ./code;
+ mkdir -p regions/EastAfrica/workspace;
+ mkdir -p regions/SouthAsia/workspace;
+ mkdir -p regions/EastAfrica/resources;
+ mkdir -p regions/SouthAsia/resources;
+ mkdir ./logs;
+ mkdir -p ./envs/credentials
+
+
+.. code-block:: bash
+
+ chmod -R g+sw <install_location>
+
+(make all files written here default to the current user group, and group-writable
+
+2: Clone code
+-------------
+(can copy-paste below as a load of commands to run on one line) Assumes you have permissions on our gitlab repo:
+
+cd into <install_location>/code
+
+.. code-block:: bash
+
+ git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-coordinator.git coordinator;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-environmental-suitability.git environmental_suitability;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-advisory-builder.git advisory_builder;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-plotting.git plotting;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-epidemiology.git epimodel;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-source-calculation.git source_gen;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-coordinator.git coordinator;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-job-flagging.git flagdir;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-met-processing.git met_extractor_v2;git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/ews-postprocessing.git post_processing;
+
+or generate this string in bash:
+
+.. code-block:: bash
+
+ for package in ews-coordinator,coordinator \
+ ews-environmental-suitability,environmental_suitability \
+ ews-advisory-builder,advisory_builder \
+ ews-plotting,plotting \
+ ews-epidemiology,epimodel \
+ ews-source-calculation,source_gen \
+ ews-coordinator,coordinator \
+ ews-job-flagging,flagdir \
+ ews-met-processing,met_extractor_v2 \
+ ews-postprocessing,post_processing;
+ do
+ repo=`echo $package | cut -d "," -f 1`
+ folder=`echo $package | cut -d "," -f 2`
+ printf "%s" "git clone git@gitlab.developers.cam.ac.uk:gilligan-epid/wheat-rusts/${repo}.git ${folder};"
+ done
+
+3: Copy the Resources needed for the various chart areas:
+---------------------------------------------------------
+.. admonition:: Configuration files from Cambridge required
+
+ Due to the size, project resources are not included in the git repository. You will need to ask the Cambridge team
+ for the latest version of the resources archive.
+
+Once you have received an archive of ``<EastAfrica/SouthAsia>_resources_<latest_date>.tar.gz`` from Cambridge, unpack
+into ``<install_location>/regions/<EastAfrica/SouthAsia>/resources``. This will place all the necessary JSON
+configuration and other assets required for the pipeline to run.
+
+.. code-block:: bash
+
+ tar -zxvf resources_<latest_date>.tar.gz
+
+Edit the coordinator run script to fix the root paths (not necessary if running in our docker image):
+
+<install_location>/code/coordinator/scripts/run_Processor.py
+
+Put a valid email credentials file into <install_location>/envs/credentials/Cred_gmail.json
+
+Example format:
+
+.. code-block:: bash
+
+ {
+ "user" : "clusternotifications@gmail.com",
+ "pass" : "passwordhere",
+ "host" : "smtp.gmail.com",
+ "port" : 587,
+ "toaddrs" : ["person1@cam.ac.uk","person1@cam.ac.uk"]
+ }
+
+4: Run the docker image:
+-----------------------
+install docker on your machine https://docs.docker.com/get-docker/
+
+Build the docker image (slow, one-off step)
+
+.. code-block:: bash
+
+ cd <install_location>/code/coordinator/configs/docker/build/
+
+By default, the buildfile will create an image with the user credentials of the Cam eEpi group server. To make a version with the right credentials to write you your filesystem, run the build script provided, but edit to override the following credentials:
+
+(from build.sh)
+
+.. code-block:: bash
+
+ sudo docker build -t ews_coordinator --build-arg GNAME=<your value> --build-arg UNAME=<your value> --build-arg UID=<your value> --build-arg GID=<your value> -f Dockerfile .
+
+Where the various variables are all LINUX user credentials (type the ‘id’ command to see all these values)
+
+Make a copy of the docker launch template file and edit the source dir to match that of your installation.
+
+.. code-block:: bash
+
+ cd <install_location>/code/coordinator/configs/docker/run
+ cp launchDocker_template.sh launchDocker_<installname>.sh
+
+Launch the docker instance (you will end up inside the docker instance in the “code†dir)
+
+.. code-block:: bash
+
+ bash launchDocker_<installname>.sh
+
+5: Run test commands within Docker:
+-----------------------------------
+You can run all the integration tests, or run “proper†commands that work from met data deployed elsewhere.
+
+Run the suite of integration tests:
+
+when inside the running docker container:
+
+.. code-block:: bash
+
+ cd <install_location>/code/coordonator/tests/integration/
+ ./run_tests.sh
+
+outputs will be in: /storage/app/EWS_prod/code/coordinator/tests/test_data/test_deployment/regions/EastAfrica/workspace
+
+6: Run a full run
+-----------------
+.. admonition:: Met Office data required
+
+ This step is to prepare for a full production deployment. You will need to have assess to data from the UK Met
+ Office available, either on the local server or another server which can be connected to with ssh.
+
+ Example data can be provided by the Cambridge team.
+
+
+SurveyData (note: day should be the day before the Depo and Env suit dates)
+
+rm -v /storage/app/EWS_prod/regions/EastAfrica/workspace/SURVEYDATA_20220501/STATUS_*;/storage/app/EWS_prod/code/coordinator/run_Processor.sh -p Survey -c /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json --noupload -s 20220807;
+
+/storage/app/EWS_dev/code/coordinator/run_dev_Processor.sh -p Survey -c /storage/app/EWS_dev/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json --noupload -s 20220807;
+
+
+Depo:
+
+rm -v /storage/app/EWS_prod/regions/EastAfrica/workspace/DEPOSITION_20220502/STATUS_*;/storage/app/EWS_prod/code/coordinator/run_Processor.sh -p Deposition -c /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json --noupload -s 20220808
+
+/storage/app/EWS_dev/code/coordinator/run_dev_Processor.sh -p Deposition -c /storage/app/EWS_dev/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json --noupload -s 20220808
+
+Env suit:
+
+rm -rf /storage/app/EWS_prod/regions/EastAfrica/workspace/ENVIRONMENT_2.0_20220502/; /storage/app/EWS_prod/code/coordinator/run_Processor.sh -p Environment -c /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json --noupload -s 20220808;
+
+rm -rf /storage/app/EWS_dev/regions/EastAfrica/workspace/ENVIRONMENT_2.0_20220808; /storage/app/EWS_dev/code/coordinator/run_dev_Processor.sh -p Environment -c /storage/app/EWS_dev/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json --noupload -s 20220808;
+
+EPI
+
+Set the start of the season be editing the following lines in <install_location>/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+
+Line 154 "CalculationSpanDays" : ["20220807",5],
+Line 225 "seasonStartString" : "20220807",
+
+rm /storage/app/EWS_prod/regions/EastAfrica/workspace/EPI_20220502/STATUS_*; /storage/app/EWS_prod/code/coordinator/run_Processor.sh -p Epidemiology -c /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json --noupload -s 20220808;
+
+[Currently it requires at least 2 days of output to work properly, throws an error with a single day]
+
+Advisory
+
+rm /storage/app/EWS_prod/regions/EastAfrica/workspace/SUMMARY_20220502/STATUS_*;
+/storage/app/EWS_prod/code/coordinator/run_Processor.sh -p Advisory -c /storage/app/EWS_prod/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json --noupload -s 20220808;
-for subsubsections
-^^^^^^^^^^^^^^^^^^
+rm /storage/app/EWS_dev/regions/EastAfrica/workspace/SUMMARY_20220502/STATUS_*;
+/storage/app/EWS_dev/code/coordinator/run_dev_Processor.sh -p Advisory -c /storage/app/EWS_dev/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json --noupload -s 20220808;
-for paragraphs
-""""""""""""""
diff --git a/docs/api.rst b/docs/api.rst
index 92c6af1..3d57073 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -6,7 +6,7 @@ API
:template: custom-module-template.rst
:recursive:
- coordinator
+.. coordinator
plotting
ews_postprocessing
met_processing
diff --git a/docs/index.rst b/docs/index.rst
index e849325..64f33ee 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -6,6 +6,8 @@ Welcome to the EWS documentation!
.. toctree::
+ :maxdepth: 1
+
Home <self>
Installation <_source/installation>
General Overview <_source/general_overview>
--
GitLab