From d445e58541d30ccc14f54fdd645267ee0ea883e3 Mon Sep 17 00:00:00 2001
From: tm689 <tm689@cam.ac.uk>
Date: Wed, 14 Jun 2023 14:07:46 +0100
Subject: [PATCH] docs: add general documentation
---
coordinator/EnvSuitPipeline.py | 23 ++++++++++++++++++----
coordinator/ProcessorEnvironment.py | 9 ++++++---
coordinator/ProcessorSurveysWRSIS.py | 2 ++
coordinator/ProcessorSurveysWRT.py | 4 +++-
coordinator/ProcessorSurveyskobotoolbox.py | 3 ++-
5 files changed, 32 insertions(+), 9 deletions(-)
diff --git a/coordinator/EnvSuitPipeline.py b/coordinator/EnvSuitPipeline.py
index 7a76e82..30f4bfa 100644
--- a/coordinator/EnvSuitPipeline.py
+++ b/coordinator/EnvSuitPipeline.py
@@ -1,3 +1,5 @@
+"""Managing the met processing pipeline for the environmental suitability run."""
+
import datetime as dt
import json
import logging
@@ -17,6 +19,8 @@ logger = logging.getLogger('Processor.pipeline')
###############################################################################
def loadConfig(configFile):
+ '''Load a json config file.'''
+
logger.info(f"Loading config file: {configFile}")
try:
@@ -30,6 +34,8 @@ def loadConfig(configFile):
def getParameter(config, parameter):
+ '''Get a parameter from a config.'''
+
logger.info(f"Getting {parameter} from the config")
try:
@@ -42,6 +48,8 @@ def getParameter(config, parameter):
def generate_temporal_points(file, datestr, timeresolution, nDaysForecast):
+ '''Create the time series excluding the end time.'''
+
datenow = dt.datetime(int(datestr[:4]), int(datestr[4:6]), int(datestr[6:8]), 0, 0)
starttime = datenow + dt.timedelta(hours=timeresolution)
@@ -63,7 +71,9 @@ def generate_temporal_points(file, datestr, timeresolution, nDaysForecast):
return outfile
-def clean(workPath): # Clean temporary files and folders from the working directory
+def clean(workPath):
+ '''Clean temporary files and folders from the working directory.'''
+
try:
logger.info(f"Clean temporary files and folders from {workPath}")
shutil.rmtree(workPath + 'extraction', ignore_errors=True)
@@ -98,15 +108,19 @@ def clean(workPath): # Clean temporary files and folders from the working direct
def run_extraction(run_params: dict, sys_params: dict):
+ '''Run weather data extraction with the :class:`met_processing.runner.common.job_runner` package.'''
+
logger.info(f"Running regridding in multi process mode.")
job_runner.run_extraction(run_params, sys_params)
logger.info('Data extracted and chunked')
def run_post_processing(run_params: dict, sys_params: dict, processor_name: str):
+ '''Run post processing with the :class:`met_processing.runner.common.job_runner` package.'''
+
logger.info(f"Running post-processing.")
job_runner.run_post_processing(run_params, sys_params, processor_name)
- logger.info('Data extracted and chunked')
+ logger.info('Data post processing is completed')
# def run_merger(run_params: dict, sys_params: dict, processor_name: str):
@@ -121,9 +135,10 @@ def run_post_processing(run_params: dict, sys_params: dict, processor_name: str)
#lawrence coment back to original (prevent_overwrite=True)
def run_pipeline(pipeline_config, region, dateString, extracted = False, prevent_overwrite = True):
'''
- The prevent_overwrite parameter can be set to False if you want to re-run
- a job in-place.
+ Call this function to run the whole :class:`met_processing` pipeline for environmental suitability.
'''
+ # The prevent_overwrite parameter can be set to False if you want to re-run a job in-place.
+
# Get parameters from the config
resourcesPath = getParameter(pipeline_config,'RESOURCES_PATH')
workPath = getParameter(pipeline_config,'WORK_PATH') + 'ENVIRONMENT_2.0_' + dateString + '/'
diff --git a/coordinator/ProcessorEnvironment.py b/coordinator/ProcessorEnvironment.py
index 7e1b04d..568e280 100644
--- a/coordinator/ProcessorEnvironment.py
+++ b/coordinator/ProcessorEnvironment.py
@@ -181,9 +181,12 @@ def process_EWS_plotting_env2_0(jobPath,config):
# Include further diseases in plotting. In this case the irrigated suitabilite for the rusts.
# TODO: move this part out into a config
extra_diseases = [
- EnvSuitDiseaseInfo("Stem rust temp-only", "stem_rust_temponly", config['StartString'], "StemRust_TempOnly", csv_template_dir),
- EnvSuitDiseaseInfo("Leaf rust temp-only", "leaf_rust_temponly", config['StartString'], "LeafRust_TempOnly", csv_template_dir),
- EnvSuitDiseaseInfo("Stripe rust temp-only", "stripe_temponly", config['StartString'], "StripeRust_TempOnly", csv_template_dir)
+ #EnvSuitDiseaseInfo("Stem rust temp-only", "stem_rust_temponly", config['StartString'], "StemRust_TempOnly", csv_template_dir),
+ #EnvSuitDiseaseInfo("Leaf rust temp-only", "leaf_rust_temponly", config['StartString'], "LeafRust_TempOnly", csv_template_dir),
+ #EnvSuitDiseaseInfo("Stripe rust temp-only", "stripe_temponly", config['StartString'], "StripeRust_TempOnly", csv_template_dir),
+ EnvSuitDiseaseInfo("Fusarium head blight 15-30-35C", "fusarium_15-30-35", config['StartString'], "FusariumHeadBlight_15-30-35", csv_template_dir),
+ EnvSuitDiseaseInfo("Fusarium head blight 15-30C", "fusarium_15-30", config['StartString'], "FusariumHeadBlight_15-30", csv_template_dir),
+ EnvSuitDiseaseInfo("Fusarium head blight 15-35C", "fusarium_15-35", config['StartString'], "FusariumHeadBlight_15-35", csv_template_dir)
]
env_suit_processor.add_diseases(diseases=extra_diseases)
diff --git a/coordinator/ProcessorSurveysWRSIS.py b/coordinator/ProcessorSurveysWRSIS.py
index 3cab5c5..92c0a12 100644
--- a/coordinator/ProcessorSurveysWRSIS.py
+++ b/coordinator/ProcessorSurveysWRSIS.py
@@ -21,6 +21,8 @@ logger = logging.getLogger('Processor.Surveys.WRSIS')
add_filters_to_sublogger(logger)
def get_from_WRSIS(form_credentials: dict, startDate: str, endDate: str):
+ '''Make the HTTP request, then checking the status code.'''
+
date_params = {
'fromDate':startDate,
'toDate':endDate}
diff --git a/coordinator/ProcessorSurveysWRT.py b/coordinator/ProcessorSurveysWRT.py
index 634eb5e..62b86da 100644
--- a/coordinator/ProcessorSurveysWRT.py
+++ b/coordinator/ProcessorSurveysWRT.py
@@ -36,6 +36,8 @@ cases_severity = {
}
def get_from_WRT(form_credentials: dict, startDate: str, endDate: str):
+ '''Make the HTTP request, then checking the status code.'''
+
url = f"{form_credentials['url']}WheatRustAPI/api/Survey"
url += '?strStopDate=' + endDate
url += '&strStartDate=' + startDate
@@ -65,7 +67,7 @@ def get_from_WRT(form_credentials: dict, startDate: str, endDate: str):
def nested_to_flattened(df):
'''WRT rust data is in a nested format, so it require to be flattened.
- To do this, the nested data need to be spareated into dedicated columns.'''
+ To do this, the nested data need to be separeated into dedicated columns.'''
# check if the dataframe is empty, if it is then add the raw columns
if len(df.index) == 0:
diff --git a/coordinator/ProcessorSurveyskobotoolbox.py b/coordinator/ProcessorSurveyskobotoolbox.py
index 67330be..1d5d96f 100644
--- a/coordinator/ProcessorSurveyskobotoolbox.py
+++ b/coordinator/ProcessorSurveyskobotoolbox.py
@@ -21,6 +21,7 @@ logger = logging.getLogger('Processor.Surveys.kobotoolbox')
add_filters_to_sublogger(logger)
def get_from_kobotoolbox(url,form_id,form_token,**kwargs):
+ '''Make the HTTP request, then checking the status code.'''
# Kenya survey form
#url = 'https://kf.kobotoolbox.org/'
@@ -43,6 +44,7 @@ def get_from_kobotoolbox(url,form_id,form_token,**kwargs):
return response
def build_dataframe(response):
+ '''Crude merging of list of dicts into pandas dataframe'''
result_count = response.json()['count']
@@ -50,7 +52,6 @@ def build_dataframe(response):
request_results = response.json()['results']
- # crude merging of list of dicts into pandas dataframe
df = DataFrame.from_records(request_results)
return df
--
GitLab