From 0854215bdf620533babc820dce963940219d8482 Mon Sep 17 00:00:00 2001
From: lb584 <lb584@cam.ac.uk>
Date: Tue, 16 Apr 2024 10:10:23 +0100
Subject: [PATCH] removing the upload code
---
coordinator/Processor.py | 100 +++++++++++++-------------
coordinator/ProcessorComponents.py | 6 +-
coordinator/ProcessorServer.py | 108 ++++++++++++++---------------
3 files changed, 107 insertions(+), 107 deletions(-)
diff --git a/coordinator/Processor.py b/coordinator/Processor.py
index 795f2d7..721d741 100755
--- a/coordinator/Processor.py
+++ b/coordinator/Processor.py
@@ -512,7 +512,7 @@ class Processor:
endJob(status, ignore_inprogress = True, premature = False)
# files and directories that will be uploaded to public server
- files_to_send = []
+ # files_to_send = []
# files and directories that will be earmarked for removal after a
# successful job
@@ -563,12 +563,12 @@ class Processor:
'output': None,
'clearup': None}
- if 'output' in proc_out.keys():
- append_item_to_list(
- proc_out['output'],
- files_to_send,
- proc_description,
- status)
+ # if 'output' in proc_out.keys():
+ # append_item_to_list(
+ # proc_out['output'],
+ # files_to_send,
+ # proc_description,
+ # status)
if 'clearup' in proc_out.keys():
append_item_to_list(
@@ -590,52 +590,52 @@ class Processor:
self.logger.info('Finished with EWS-Plotting, appending images to list for transfer')
- if ews_plotting_outputs:
- append_item_to_list(
- ews_plotting_outputs,
- files_to_send,
- proc_description,
- status)
+ # if ews_plotting_outputs:
+ # append_item_to_list(
+ # ews_plotting_outputs,
+ # files_to_send,
+ # proc_description,
+ # status)
self.logger.info(f'Finished with config {configIndex + 1} of {config_paths_length}')
- # send results to remote server
-
- if not noupload:
- try:
- ProcessorComponents.upload(universal_config, files_to_send, component)
- except IndexError:
- status.reset('WARNING')
-
- except:
- self.logger.exception('Failed to upload files to remote server')
- status.reset('ERROR')
- endJob(status, premature = True)
-
- # check if there is a second location on willow to provide results
- if 'ServerPathExtra' in configjson[component]:
-
- self.logger.info('There is an extra path to send results to:')
-
- extra_path = configjson[component]['ServerPathExtra']
-
- self.logger.info(extra_path)
-
- universal_config_extra = universal_config.copy()
- universal_config_extra['ServerPath'] = extra_path
-
- try:
- ProcessorComponents.upload(universal_config_extra, files_to_send, component)
- except IndexError:
- status.reset('WARNING')
-
- except:
- self.logger.exception('Failed to upload files to extra directory on remote server')
- status.reset('ERROR')
- endJob(status, premature = True)
-
- else:
- self.logger.info('Because noupload argument was present, not sending results to remote server')
+ # # send results to remote server
+ #
+ # if not noupload:
+ # try:
+ # ProcessorComponents.upload(universal_config, files_to_send, component)
+ # except IndexError:
+ # status.reset('WARNING')
+ #
+ # except:
+ # self.logger.exception('Failed to upload files to remote server')
+ # status.reset('ERROR')
+ # endJob(status, premature = True)
+ #
+ # # check if there is a second location on willow to provide results
+ # if 'ServerPathExtra' in configjson[component]:
+ #
+ # self.logger.info('There is an extra path to send results to:')
+ #
+ # extra_path = configjson[component]['ServerPathExtra']
+ #
+ # self.logger.info(extra_path)
+ #
+ # universal_config_extra = universal_config.copy()
+ # universal_config_extra['ServerPath'] = extra_path
+ #
+ # try:
+ # ProcessorComponents.upload(universal_config_extra, files_to_send, component)
+ # except IndexError:
+ # status.reset('WARNING')
+ #
+ # except:
+ # self.logger.exception('Failed to upload files to extra directory on remote server')
+ # status.reset('ERROR')
+ # endJob(status, premature = True)
+ #
+ # else:
+ # self.logger.info('Because noupload argument was present, not sending results to remote server')
status.reset('SUCCESS')
diff --git a/coordinator/ProcessorComponents.py b/coordinator/ProcessorComponents.py
index 2eb745b..aa8b519 100644
--- a/coordinator/ProcessorComponents.py
+++ b/coordinator/ProcessorComponents.py
@@ -12,10 +12,10 @@ from typing import List
# coordinator stages: pre, in (during) and plotting.
-from ProcessorServer import (
+#from ProcessorServer import (
# process_pre_job_server_download,
- upload
-)
+ #upload
+#)
from ProcessorUtils import (
add_filters_to_sublogger,
diff --git a/coordinator/ProcessorServer.py b/coordinator/ProcessorServer.py
index 6a93375..2e13b03 100644
--- a/coordinator/ProcessorServer.py
+++ b/coordinator/ProcessorServer.py
@@ -107,60 +107,60 @@ add_filters_to_sublogger(logger)
#
# return True
-def upload(config,FilesToSend,component):
-
- usual_path = f"{config['StartString']}_0000/"
-
- component_path = {
- 'Environment' : usual_path,
- 'Deposition' : usual_path,
- 'Epidemiology' : usual_path,
- 'Survey' : f"SURVEYDATA_{config['StartString']}_0000/",
- 'Scraper' : usual_path,
- 'MetResample' : f"MET_RESAMPLE_{config['StartString']}_0000/",
- 'Advisory' : usual_path }
-
-
- # TODO: make path discern Daily or Weekly sub-directory
-
- OutputServerPath = f"{config['ServerPath']}/{component_path[component]}"
-
- logger.info(f"Trying upload to {config['ServerName']}:{OutputServerPath}")
-
- logger.info(f"File(s) that will be put on remote server: {FilesToSend}")
-
- if len(FilesToSend) == 0:
- logger.warning('No files to send, so skipping this task')
- raise IndexError
-
- logger.debug("Making path directory on remote server if it doesn't already exist")
-
- server_key = config['ServerKey']
- if server_key == "":
- ssh_cmd = f"mkdir -p {OutputServerPath}"
- run_in_shell: bool = True
- else:
- ssh_cmd = ["ssh", "-i", server_key, "-o", "StrictHostKeyChecking=no", config['ServerName'],
- f"mkdir -p {OutputServerPath}"]
- run_in_shell: bool = False
-
- description_short = 'upload ssh'
- description_long = 'make remote directory'
- subprocess_and_log(ssh_cmd, description_short, description_long, shell=run_in_shell)
-
- logger.debug('Sending file(s) to remote server')
-
- if server_key == "":
- scp_cmd = ["scp", "-r", *FilesToSend, OutputServerPath]
- else:
- scp_cmd = ["scp", "-ri", server_key, "-o", "StrictHostKeyChecking=no", *FilesToSend,
- f"{config['ServerName']}:{OutputServerPath}"]
-
- description_short = 'upload scp'
- description_long = 'scp files to remote directory'
- subprocess_and_log(scp_cmd, description_short, description_long)
-
- return
+# def upload(config,FilesToSend,component):
+#
+# usual_path = f"{config['StartString']}_0000/"
+#
+# component_path = {
+# 'Environment' : usual_path,
+# 'Deposition' : usual_path,
+# 'Epidemiology' : usual_path,
+# 'Survey' : f"SURVEYDATA_{config['StartString']}_0000/",
+# 'Scraper' : usual_path,
+# 'MetResample' : f"MET_RESAMPLE_{config['StartString']}_0000/",
+# 'Advisory' : usual_path }
+#
+#
+# # TODO: make path discern Daily or Weekly sub-directory
+#
+# OutputServerPath = f"{config['ServerPath']}/{component_path[component]}"
+#
+# logger.info(f"Trying upload to {config['ServerName']}:{OutputServerPath}")
+#
+# logger.info(f"File(s) that will be put on remote server: {FilesToSend}")
+#
+# if len(FilesToSend) == 0:
+# logger.warning('No files to send, so skipping this task')
+# raise IndexError
+#
+# logger.debug("Making path directory on remote server if it doesn't already exist")
+#
+# server_key = config['ServerKey']
+# if server_key == "":
+# ssh_cmd = f"mkdir -p {OutputServerPath}"
+# run_in_shell: bool = True
+# else:
+# ssh_cmd = ["ssh", "-i", server_key, "-o", "StrictHostKeyChecking=no", config['ServerName'],
+# f"mkdir -p {OutputServerPath}"]
+# run_in_shell: bool = False
+#
+# description_short = 'upload ssh'
+# description_long = 'make remote directory'
+# subprocess_and_log(ssh_cmd, description_short, description_long, shell=run_in_shell)
+#
+# logger.debug('Sending file(s) to remote server')
+#
+# if server_key == "":
+# scp_cmd = ["scp", "-r", *FilesToSend, OutputServerPath]
+# else:
+# scp_cmd = ["scp", "-ri", server_key, "-o", "StrictHostKeyChecking=no", *FilesToSend,
+# f"{config['ServerName']}:{OutputServerPath}"]
+#
+# description_short = 'upload scp'
+# description_long = 'scp files to remote directory'
+# subprocess_and_log(scp_cmd, description_short, description_long)
+#
+# return
def get_data_from_server(jobPath,config,component):
--
GitLab