From d4c81ff641cd66892625b3f22d2a29d78c011143 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Tue, 8 Nov 2022 16:01:16 +0000
Subject: [PATCH] changing the scp logic and the ssh file checking logic to use
a different command if no server name is set
---
coordinator/ProcessorDeposition.py | 9 ++++---
coordinator/ProcessorServer.py | 14 ++++++++--
coordinator/ProcessorUtils.py | 3 ++-
.../configs/config_EastAfrica_fc_live.json | 2 +-
.../coordinator/configs/fake_ssh_key | 27 -------------------
5 files changed, 20 insertions(+), 35 deletions(-)
delete mode 100644 tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/fake_ssh_key
diff --git a/coordinator/ProcessorDeposition.py b/coordinator/ProcessorDeposition.py
index cd48605..07bcd00 100644
--- a/coordinator/ProcessorDeposition.py
+++ b/coordinator/ProcessorDeposition.py
@@ -37,11 +37,12 @@ def process_in_job_dep(jobPath,status,config,component):
# TODO: perform ssh file transfer in python instead of subprocess
server_name: str = config['ServerName']
- machine_seperator: str = ":"
- if not server_name:
- machine_seperator = ""
+ if server_name == "":
+ cmd_scp = ["scp", f"{file_path}/{file_name}.tar.gz", jobPath]
+ else:
+ cmd_scp = ["scp", "-i", config['ServerKey'], "-o", "StrictHostKeyChecking=no",
+ f"{server_name}:{file_path}/{file_name}.tar.gz", jobPath]
- cmd_scp = ["scp", "-i", config['ServerKey'], "-o", "StrictHostKeyChecking=no", f"{server_name}{machine_seperator}{file_path}/{file_name}.tar.gz", jobPath]
description_short = 'dep scp'
description_long = 'scp from server to job directory'
subprocess_and_log(cmd_scp, description_short, description_long)
diff --git a/coordinator/ProcessorServer.py b/coordinator/ProcessorServer.py
index cca3e76..d925474 100644
--- a/coordinator/ProcessorServer.py
+++ b/coordinator/ProcessorServer.py
@@ -41,11 +41,21 @@ def process_pre_job_server_download(input_args: dict):
timenow = datetime.datetime.now(tz=datetime.timezone.utc).time()
- cmd_ssh = ["ssh","-i",config['ServerKey'],"-o","StrictHostKeyChecking=no",config['ServerName'],f"test -f {file_path}/{file_name}.tar.gz"]
+ server_name: str = config['ServerName']
+ full_file_path = f"{file_path} / {file_name}.tar.gz"
+ if server_name == "":
+ cmd_check_file = [f"test -f {full_file_path} && tar -tzf {full_file_path} > /dev/null"]
+ run_in_shell: bool = True
+ else:
+ cmd_check_file = ["ssh", "-i", config['ServerKey'], "-o", "StrictHostKeyChecking=no", server_name,
+ f"test -f {full_file_path} && tar -tzf {full_file_path} >/dev/null"]
+ run_in_shell: bool = False
+
description_short = 'subprocess_ssh'
description_long = f"Checking for existence of {file_path}/{file_name}.tar.gz"
- status = subprocess_and_log(cmd_ssh,description_short,description_long,check=False)
+ status = subprocess_and_log(cmd_check_file, description_short, description_long, check = False,
+ shell = run_in_shell)
if status.returncode == 1:
diff --git a/coordinator/ProcessorUtils.py b/coordinator/ProcessorUtils.py
index 8ddfa4b..1b7d017 100644
--- a/coordinator/ProcessorUtils.py
+++ b/coordinator/ProcessorUtils.py
@@ -125,7 +125,7 @@ def get_only_existing_globs(file_globs,inplace=True):
globs_out += [fg]
return globs_out
-def subprocess_and_log(cmd,description_short,description_long,check=True,log_type='error',**kwargs):
+def subprocess_and_log(cmd,description_short,description_long,check=True,log_type='error', shell: bool = False, **kwargs):
'''Run a shell command (described by a comma separated list) and send stdout
and stderr to logfile, and raise any exception.
@@ -140,6 +140,7 @@ def subprocess_and_log(cmd,description_short,description_long,check=True,log_typ
process = subprocess.run(
cmd,
check=check,
+ shell=shell,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
**kwargs)
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
index fb841c1..3e5c32f 100644
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
+++ b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/config_EastAfrica_fc_live.json
@@ -8,7 +8,7 @@
"ResourcesPath" : "../test_data/test_deployment/regions/EastAfrica/resources/",
"ServerPath" : "/storage/moved/Ethiopia/",
"ServerName" : "ewsmanager@willow.csx.cam.ac.uk",
- "ServerKey" : "../test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/fake_ssh_key",
+ "ServerKey" : "not used, set to empty string in tests",
"Survey" : {
"ProcessPreJob" : "process_pre_job_survey",
"ProcessInJob" : "process_in_job_survey",
diff --git a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/fake_ssh_key b/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/fake_ssh_key
deleted file mode 100644
index 303ef26..0000000
--- a/tests/test_data/test_deployment/regions/EastAfrica/resources/coordinator/configs/fake_ssh_key
+++ /dev/null
@@ -1,27 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEogIBAAKCAQEAj68M3gPjV7KdNOKWB9ZRolAtVcsUXt1cxmgp3pQEkNbtQdda
-BBt5q1GCfJIdHjJrfgKbd3h86bYERl4EwLgMJJKRGW20jGcEm5FDwEn//NLCmOGt
-36rSYiVpgakqK9jHULnq993qpid6PQMPVwAWnnZeEoU9ZAqWEpx06w5S4uwUVZi2
-J+FHNVDoGQOWE95wFwUg/qd34fPZBA5/JuuGSPUdfz2gAoIXPogooGeUcXjvOO8J
-WZkE0eeH0LnatHkcXfl/MXq35CtrsOKFkmSz881ha+D1X+UdhVYW0+J+qvwEIxNi
-sV06GWCAidJWyF0cL8N7VymgcrFt2fBZ0i0kEQIDAQABAoIBABh53VtElkmuJ5GG
-t8AGBkn8Fnzy5zlxxAgYXmmIFW8G7cu1f6QLeiJw9CV8TnGCbUU99HXZlI3Lb9b8
-1RpvP8JLJWSLh6ITOnm1uTDcajsLPBMG/rOHSedmMB7iTkjB2NIrmZc3um0rdGtI
-Riiv3yqHR2coQ1jzZLUu6cvQ2NXG2Bsxw3bXh6pIU0wD8iwrG6kM68AgEu4qFNze
-e+5Y3s9rqFfCwccQgSZUolmJaYBQQ8CVTguOaxyMPgrVLELxDaZlWUOjKmA3Z6og
-1lbx83P9SeXQdwOVo3kFQXwjnTyjhELhFjawozcYrjDsbIzjcJlmBWNEUoVbMBgZ
-ebtAkvECgYEAx4n/0io5CUh4WRux3CkAsfpOP+niy9UnGf4YulYGpzwWVaNi7edj
-6CrI2V/qcNOJi7OTv0yqldo5+dVeQx0zJIcuPc6p+n7QoiM/XHtCtEanXmFYcWrW
-wDbedQ8F1VEKznIb6zfuWKsHv2srdbdfkAyfL1/EVd96sdKA8rqryrcCgYEAuFcQ
-zcenuc1LFbkvpY1e5KHiJ4AuLPY2bxCF8IyKrj8rd4W/JLfpnxkeOkoIlKwcm1xs
-eRW56BDqrLr8Z276AfD1RcQfwlz3i5rVgYRG6IR/fZqrE4DT3dd8QJoX+Zp/izba
-FxQDbcYg3YRnpd/rKyqaX2HY6dwvuvHYG9yoX3cCgYBM8rm1mVS/cLQAYP640RMU
-j1tHqHEue7eFfYiZqv0L1s8iHUzUQ6wyZvVnZmapNdpQBmNOnUkXWr2Ky657ITKM
-vpileyHvmKHqNL+FcX8quCho7A9IRZQlMcOdzrpIJeWN8VE05y/B/8qQFcr/rURr
-74IfqcWz++d71ZqmgDgphwKBgG9TjX9pM18fn1foNeE6ZnOXXHhUjg5zR1590RX8
-Goiu97ZAHLa1Jy8BQWMccOAwrToacBwURejjxoY4itSAjhX8CnRuxGtiL9Gs2YJV
-e2lTE5esf+LQa6NPTufI5+/DIYHWx5EmcgeqnxCItPvVJjY0jXRLKG/bnSu+gLAa
-68hzAoGAGGJ2R5aZavVIU8r0ECvSkUyG8g/pVzbJl1kK+MkLDqMLzGudForQtkx9
-9KmZ253qDkRfiWFsOH6y5dUUgSNb0S+XkPr1OzJXkIJF/hy3ZYCJOUxGZ+fLsUxd
-Xl26l7K7B6tImGOYKg3u04zeYX/60w7b3+hOSs5q0jwf1dIKCfo=
------END RSA PRIVATE KEY-----
\ No newline at end of file
--
GitLab