From b8ee59e96999cd30468e978fa183fb153ff4e000 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Wed, 9 Nov 2022 11:56:20 +0000
Subject: [PATCH] more work swapping ssh and scp commands for when run in local
mode
---
coordinator/ProcessorEnvironment.py | 9 ++++----
coordinator/ProcessorServer.py | 21 ++++++++++++++-----
.../configs/config_EastAfrica_fc_live.json | 6 +++---
3 files changed, 24 insertions(+), 12 deletions(-)
diff --git a/coordinator/ProcessorEnvironment.py b/coordinator/ProcessorEnvironment.py
index e39386e..5e96960 100644
--- a/coordinator/ProcessorEnvironment.py
+++ b/coordinator/ProcessorEnvironment.py
@@ -40,11 +40,12 @@ def process_in_job_env2_0(jobPath,status,config,component):
# TODO: perform ssh file transfer in python instead of subprocess
server_name: str = config['ServerName']
- machine_seperator: str = ":"
- if not server_name:
- machine_seperator = ""
+ if server_name == "":
+ cmd_scp: list = ["scp", f"{file_path}/{file_name}.tar.gz", jobPath]
+ else:
+ cmd_scp: list = ["scp", "-i", config['ServerKey'], "-o", "StrictHostKeyChecking=no",
+ f"{config['ServerName']}:{file_path}/{file_name}.tar.gz", jobPath]
- cmd_scp: list = ["scp","-i",config['ServerKey'],"-o","StrictHostKeyChecking=no",f"{config['ServerName']}{machine_seperator}{file_path}/{file_name}.tar.gz", jobPath]
description_short = 'env2 scp'
description_long = 'Copying file from remote server to job directory'
# lawrence comment in/out
diff --git a/coordinator/ProcessorServer.py b/coordinator/ProcessorServer.py
index d925474..50a68f1 100644
--- a/coordinator/ProcessorServer.py
+++ b/coordinator/ProcessorServer.py
@@ -42,7 +42,7 @@ def process_pre_job_server_download(input_args: dict):
timenow = datetime.datetime.now(tz=datetime.timezone.utc).time()
server_name: str = config['ServerName']
- full_file_path = f"{file_path} / {file_name}.tar.gz"
+ full_file_path = f"{file_path}/{file_name}.tar.gz"
if server_name == "":
cmd_check_file = [f"test -f {full_file_path} && tar -tzf {full_file_path} > /dev/null"]
run_in_shell: bool = True
@@ -57,7 +57,7 @@ def process_pre_job_server_download(input_args: dict):
status = subprocess_and_log(cmd_check_file, description_short, description_long, check = False,
shell = run_in_shell)
- if status.returncode == 1:
+ if status.returncode > 0:
# a time check in UTC. If it's late, raise warning, if very late, raise error
@@ -122,15 +122,26 @@ def upload(config,FilesToSend,component):
logger.debug("Making path directory on remote server if it doesn't already exist")
- ssh_cmd = ["ssh","-i",config['ServerKey'],"-o","StrictHostKeyChecking=no",config['ServerName'], f"mkdir -p {OutputServerPath}"]
+ server_key = config['ServerKey']
+ if server_key == "":
+ ssh_cmd = ["ssh", f"mkdir -p {OutputServerPath}"]
+ run_in_shell: bool = True
+ else:
+ ssh_cmd = ["ssh", "-i", server_key, "-o", "StrictHostKeyChecking=no", config['ServerName'],
+ f"mkdir -p {OutputServerPath}"]
+ run_in_shell: bool = False
description_short = 'upload ssh'
description_long = 'make remote directory'
- subprocess_and_log(ssh_cmd, description_short, description_long)
+ subprocess_and_log(ssh_cmd, description_short, description_long, shell=run_in_shell)
logger.debug('Sending file(s) to remote server')
- scp_cmd = ["scp","-ri",config['ServerKey'],"-o","StrictHostKeyChecking=no",*FilesToSend, f"{config['ServerName']}:{OutputServerPath}"]
+ if server_key == "":
+ scp_cmd = ["scp", *FilesToSend, {OutputServerPath}]
+ else:
+ scp_cmd = ["scp", "-ri", server_key, "-o", "StrictHostKeyChecking=no", *FilesToSend,
+ f"{config['ServerName']}:{OutputServerPath}"]
description_short = 'upload scp'
description_long = 'scp files to remote directory'
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
index 3e5c32f..366f6a6 100644
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
@@ -148,7 +148,7 @@
},
"Epidemiology" : {
"DiseaseNames" : ["StemRust"],
- "CalculationSpanDays" : ["20220501",5],
+ "CalculationSpanDays" : [0,1],
"TimeStep_hours": "3",
"ProcessPreJob" : "process_pre_job_epi",
"ProcessInJob" : "process_in_job_epi",
@@ -220,7 +220,7 @@
"ProcessPreJob" : "query_past_successes",
"ProcessInJob" : "process_in_job_advisory",
"ProcessEWSPlotting" : "do_nothing",
- "seasonStartString" : "20220501",
+ "seasonStartString" : "20210901",
"Environment" : {
"SuccessFileTemplate" : "${WorkspacePath}ENVIRONMENT_2.0_${StartString}/STATUS_SUCCESS",
"DataPathTemplate" : "${WorkspacePath}/ENVIRONMENT_2.0_${dateString}/plotting/${SubRegionNameLower}/input_csvs/",
@@ -270,7 +270,7 @@
}
},
"Surveys" : {
- "variety_names_tidy_fn" : "/media/scratch/lb584_scratch/projects/ews_local_prod/regions/EastAfrica/resources/advisory_builder/configs/config_EastAfrica_variety_names_tidy.json",
+ "variety_names_tidy_fn" : "../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/configs/config_EastAfrica_variety_names_tidy.json",
"CountryColumnName" : "surveyor_infromation-country",
"EastAfrica" : {
"ShapeFilenameAdmin0" : "../test_data/test_deployment/regions/EastAfrica/resources/advisory_builder/assets/EthKen_admin0.shp",
--
GitLab