diff --git a/DataProcessor.pl b/DataProcessor.pl index 7c118d5a5a39ee3a6f8022ce220f9eb2392bb32e..bfa2f26de79b31ddf0b9f9de90b4456d0fc07bf8 100755 --- a/DataProcessor.pl +++ b/DataProcessor.pl @@ -23,7 +23,8 @@ my $debugTimeString = getTimestampNow(); my $debugOutput = 0; my $debugNoFTPDownload = 0; -my $debugNoUpload = 0; +my $debugNoUpload = 1; +my $debugNoPlot = 1; # read in project configuration from input argument my $configFilePath = shift; @@ -42,8 +43,10 @@ my $todayString = DateTime->today(time_zone => "UTC")->strftime("%Y%m%d"); my $jobIDString = "DEPOSITION"; -my $todayFolderPath = $workspacePath."/".$todayString; +my $todayFolderPath = $workspacePath."/DEPOSITION_".$todayString; +my $NAMEDirectoryPrefix = get_NAME_directory_prefix(); +my @NAMEFilePrefixes = get_NAME_file_prefixes(); my @maintainersCam = ("jws52\@cam.ac.uk", "tm689\@cam.ac.uk", "rs481\@cam.ac.uk"); my @maintainersMO = ("william.thurston\@metoffice.gov.uk"); @@ -228,14 +231,14 @@ beginRun(); #Enter destination for the files: chdir($todayFolderPath) or handleError("ERROR: Unable to change working directory to output folder path :".$todayFolderPath); -my $todayOutputDir = "NAME_wheatrust_".$todayString; +my $todayOutputDir = $NAMEDirectoryPrefix.$todayString; my $todayOutputFileName = $todayOutputDir.".tar.gz"; unless($debugNoFTPDownload) { # check if Met Office have put today's environment results onto willow. # If file is not there, script will not die, instead it prints warning to logfile - system("scp -i ".$coordinatorPath."/ssh_key_willow willow.csx.cam.ac.uk:/storage/sftp/metofficeupload/upload/".$projectName."/fromMO/".$todayOutputFileName." ".$todayFolderPath."/"); + system("scp -i ".$coordinatorPath."/ssh_key_willow willow.csx.cam.ac.uk:/storage/sftp/metofficeupload/upload/".$projectName."/fromMO/daily_name/".$todayOutputFileName." ".$todayFolderPath."/"); if($debugOutput && -e $todayOutputFileName) { print("DEBUG: ".$debugTimeString." ".$todayOutputFileName." obtained from willow sftp directory"); @@ -289,7 +292,7 @@ unless(-d $todayOutputDir) { #Rename the folder for Dan: my $oldTodayOutputDir = $todayOutputDir; -$todayOutputDir = "NAME_wheatrust_".$todayString; +$todayOutputDir = $NAMEDirectoryPrefix.$todayString; unless($oldTodayOutputDir eq $todayOutputDir) {#They will only differ if we're using a set of testing NAME output dircopy($oldTodayOutputDir, $todayOutputDir) or handleError("ERROR: Unable to copy directory from ".$oldTodayOutputDir." to ".$todayOutputDir); @@ -297,11 +300,10 @@ unless($oldTodayOutputDir eq $todayOutputDir) {#They will only differ if we're u #Check if the zip actually had output inside it - maininput.txt and at least one file of each depositionEthiopia_Stem_*.txt, depositionEthiopia_Stripe_*.txt -my @necessaryOutputs = ( - $todayOutputDir."/maininput.txt", - $todayOutputDir."/deposition".$projectName."_Stem_C1_T1_".$todayString."0300.txt", - $todayOutputDir."/deposition".$projectName."_Stripe_C1_T1_".$todayString."0300.txt" - ); +my @necessaryOutputs = ($todayOutputDir."/maininput.txt"); +foreach my $NAMEFilePrefix (@NAMEFilePrefixes) { + push @necessaryOutputs, $todayOutputDir."/".$NAMEFilePrefix."C1_T1_".$todayString."0300.txt"; +} foreach my $necessaryOutput (@necessaryOutputs) { unless(-e $necessaryOutput) { @@ -311,7 +313,30 @@ foreach my $necessaryOutput (@necessaryOutputs) { my $plotPath = $projectPath."/Plotting/"; +#Bung all necessary outputs into folder named $todayString_0000 here in a nice structure +my $localOutputFolderName = $todayString."_0000"; + #Plotting: +if($debugNoPlot) { + #handleError("No plotting taking place", \@maintainersCam, $severityStringWARNING); + print("DEBUG: No plotting taking place"); + + #Successful run, no plotting expected at beginning of Feb 2020. + # placeholder file state this + my $noplotFilePath = $todayFolderPath."/".$jobIDString."_NO_PLOTTING.txt"; + open(my $noplotFH, ">", $noplotFilePath) or handleError("ERROR: Unable to open confirmation file for writing: ".$doneFilePath); + print $noplotFH "Completed: ".getTimestampNow()."\n"; + close($noplotFH) or handleError("ERROR: Unable to write ".$noplotFilePath); + + #Flag overall success of operation: + open(my $successFH, ">", $doneFilePath) or handleError("ERROR: Unable to open confirmation file for writing: ".$doneFilePath); + print $successFH "Completed: ".getTimestampNow()."\n"; + close($successFH) or handleError("ERROR: Unable to write ".$doneFilePath); + + #Remove the in progress file and exit: + terminateRun(); +} + chdir($plotPath) or handleError("ERROR: Unable to change to plotting directory ".$plotPath); system("export R_LIBS=/home/ewsmanager/R-packages-EWS-clustering/x86_64-pc-linux-gnu-library/3.5; /usr/local/R/bin/Rscript Submit_mass_Gilligan.R config/ds603/NAME_plotting.json --fastanimation --NAME; unset R_LIBS");#Note that these arguments are ordered and it will not work if they are the wrong way round... @@ -319,9 +344,6 @@ system("export R_LIBS=/home/ewsmanager/R-packages-EWS-clustering/x86_64-pc-linux #Plotting outputs in: my $plotImagePath = $plotPath."/animation_images/output/NAMEOUTPUT/".$todayString."0000/"; -#Bung all necessary outputs into folder named $todayString_0000 here in a nice structure -my $localOutputFolderName = $todayString."_0000"; - #back to the destination for the files: chdir($todayFolderPath) or handleError("ERROR: Unable to change working directory to output folder path :".$todayFolderPath); @@ -391,7 +413,6 @@ my $sourceFolder = $plotPath."/output/NAMEOUTPUT/".$todayString."0000/"; my $destFolder = $epiDepFolderName; dircopy($sourceFolder, $destFolder) or handleError("ERROR: Unable to copy directory from ".$sourceFolder." to ".$destFolder); - #TODO: Bung todays workspace folder onto the RCS