diff --git a/coordinator/ProcessorServer.py b/coordinator/ProcessorServer.py index 68ea5dd5dd38dc6c8d1095908b1762ee8ccc9ef1..3593fdd97159dda28130023f9931021916f1bafa 100644 --- a/coordinator/ProcessorServer.py +++ b/coordinator/ProcessorServer.py @@ -3,6 +3,8 @@ import datetime import logging +import os +import tarfile from string import Template from typing import List @@ -43,21 +45,25 @@ def process_pre_job_server_download(input_args: dict): server_name: str = config['ServerName'] full_file_path = f"{file_path}/{file_name}.tar.gz" - if server_name == "": - cmd_check_file = [f"test -f {full_file_path} && tar -tzf {full_file_path} > /dev/null"] - run_in_shell: bool = True + + """ + check the incoming met data tar exists and is a valid tar file (on either remote or local machine) + """ + if server_name == "": # means the file is local + data_is_ready = os.path.exists(full_file_path) and tarfile.is_tarfile(full_file_path) else: cmd_check_file = ["ssh", "-i", config['ServerKey'], "-o", "StrictHostKeyChecking=no", server_name, f"test -f {full_file_path} && tar -tzf {full_file_path} >/dev/null"] run_in_shell: bool = False - description_short = 'subprocess_ssh' - description_long = f"Checking for existence of {file_path}/{file_name}.tar.gz" + description_short = 'subprocess_ssh' + description_long = f"Checking for existence of {file_path}/{file_name}.tar.gz" - status = subprocess_and_log(cmd_check_file, description_short, description_long, check = False, - shell = run_in_shell) + status = subprocess_and_log(cmd_check_file, description_short, description_long, check = False, + shell = run_in_shell) + data_is_ready = status.returncode > 0 - if status.returncode > 0: + if not data_is_ready: # a time check in UTC. If it's late, raise warning, if very late, raise error @@ -91,7 +97,7 @@ def process_pre_job_server_download(input_args: dict): logger.info(message) endScript(premature=True) - elif status.returncode == 0: + elif not data_is_ready: logger.info(f"Data is available for config {i+1} of {len(config_paths)}, calculation shall proceed") return True