Made deposition data check configurable

DataProcessor.pl

@@ -23,7 +23,8 @@ my $debugTimeString = getTimestampNow();
 my $debugOutput = 0;
 
 my $debugNoFTPDownload = 0;
-my $debugNoUpload = 0;
+my $debugNoUpload = 1;
+my $debugNoPlot = 1;
 
 # read in project configuration from input argument
 my $configFilePath = shift;
@@ -42,8 +43,10 @@ my $todayString = DateTime->today(time_zone => "UTC")->strftime("%Y%m%d");
 
 my $jobIDString = "DEPOSITION";
 
-my $todayFolderPath = $workspacePath."/".$todayString;
+my $todayFolderPath = $workspacePath."/DEPOSITION_".$todayString;
 
+my $NAMEDirectoryPrefix = get_NAME_directory_prefix();
+my @NAMEFilePrefixes = get_NAME_file_prefixes();
 
 my @maintainersCam = ("jws52\@cam.ac.uk", "tm689\@cam.ac.uk", "rs481\@cam.ac.uk");
 my @maintainersMO = ("william.thurston\@metoffice.gov.uk");
@@ -228,14 +231,14 @@ beginRun();
 #Enter destination for the files:
 chdir($todayFolderPath) or handleError("ERROR: Unable to change working directory to output folder path :".$todayFolderPath);
 
-my $todayOutputDir = "NAME_wheatrust_".$todayString;
+my $todayOutputDir = $NAMEDirectoryPrefix.$todayString;
 my $todayOutputFileName = $todayOutputDir.".tar.gz";
 
 unless($debugNoFTPDownload) {
 
 	# check if Met Office have put today's environment results onto willow.
 	# If file is not there, script will not die, instead it prints warning to logfile
-	system("scp -i ".$coordinatorPath."/ssh_key_willow willow.csx.cam.ac.uk:/storage/sftp/metofficeupload/upload/".$projectName."/fromMO/".$todayOutputFileName." ".$todayFolderPath."/");
+	system("scp -i ".$coordinatorPath."/ssh_key_willow willow.csx.cam.ac.uk:/storage/sftp/metofficeupload/upload/".$projectName."/fromMO/daily_name/".$todayOutputFileName." ".$todayFolderPath."/");
 	
 	if($debugOutput && -e $todayOutputFileName) {
 		print("DEBUG: ".$debugTimeString." ".$todayOutputFileName." obtained from willow sftp directory");
@@ -289,7 +292,7 @@ unless(-d $todayOutputDir) {
 
 #Rename the folder for Dan:
 my $oldTodayOutputDir = $todayOutputDir;
-$todayOutputDir = "NAME_wheatrust_".$todayString;
+$todayOutputDir = $NAMEDirectoryPrefix.$todayString;
 
 unless($oldTodayOutputDir eq $todayOutputDir) {#They will only differ if we're using a set of testing NAME output
 	dircopy($oldTodayOutputDir, $todayOutputDir) or handleError("ERROR: Unable to copy directory from ".$oldTodayOutputDir." to ".$todayOutputDir);
@@ -297,11 +300,10 @@ unless($oldTodayOutputDir eq $todayOutputDir) {#They will only differ if we're u
 
 
 #Check if the zip actually had output inside it - maininput.txt and at least one file of each depositionEthiopia_Stem_*.txt, depositionEthiopia_Stripe_*.txt
-my @necessaryOutputs = (
-	$todayOutputDir."/maininput.txt", 
-	$todayOutputDir."/deposition".$projectName."_Stem_C1_T1_".$todayString."0300.txt", 
-	$todayOutputDir."/deposition".$projectName."_Stripe_C1_T1_".$todayString."0300.txt"
-	);
+my @necessaryOutputs = ($todayOutputDir."/maininput.txt");
+foreach my $NAMEFilePrefix (@NAMEFilePrefixes) {
+    push @necessaryOutputs, $todayOutputDir."/".$NAMEFilePrefix."C1_T1_".$todayString."0300.txt";
+}
 
 foreach my $necessaryOutput (@necessaryOutputs) {
 	unless(-e $necessaryOutput) {
@@ -311,7 +313,30 @@ foreach my $necessaryOutput (@necessaryOutputs) {
 
 my $plotPath = $projectPath."/Plotting/";
 
+#Bung all necessary outputs into folder named $todayString_0000 here in a nice structure
+my $localOutputFolderName = $todayString."_0000";
+
 #Plotting:
+if($debugNoPlot) {
+	#handleError("No plotting taking place", \@maintainersCam, $severityStringWARNING);
+	print("DEBUG: No plotting taking place");
+	
+	#Successful run, no plotting expected at beginning of Feb 2020. 
+	# placeholder file state this
+	my $noplotFilePath = $todayFolderPath."/".$jobIDString."_NO_PLOTTING.txt";
+	open(my $noplotFH, ">", $noplotFilePath) or handleError("ERROR: Unable to open confirmation file for writing: ".$doneFilePath);
+	print $noplotFH "Completed: ".getTimestampNow()."\n";
+	close($noplotFH) or handleError("ERROR: Unable to write ".$noplotFilePath);
+
+	#Flag overall success of operation:
+	open(my $successFH, ">", $doneFilePath) or handleError("ERROR: Unable to open confirmation file for writing: ".$doneFilePath);
+	print $successFH "Completed: ".getTimestampNow()."\n";
+	close($successFH) or handleError("ERROR: Unable to write ".$doneFilePath);
+
+	#Remove the in progress file and exit:
+	terminateRun();
+}
+
 chdir($plotPath) or handleError("ERROR: Unable to change to plotting directory ".$plotPath);
 
 system("export R_LIBS=/home/ewsmanager/R-packages-EWS-clustering/x86_64-pc-linux-gnu-library/3.5; /usr/local/R/bin/Rscript Submit_mass_Gilligan.R config/ds603/NAME_plotting.json --fastanimation --NAME; unset R_LIBS");#Note that these arguments are ordered and it will not work if they are the wrong way round... 
@@ -319,9 +344,6 @@ system("export R_LIBS=/home/ewsmanager/R-packages-EWS-clustering/x86_64-pc-linux
 #Plotting outputs in:
 my $plotImagePath = $plotPath."/animation_images/output/NAMEOUTPUT/".$todayString."0000/";
 
-#Bung all necessary outputs into folder named $todayString_0000 here in a nice structure
-my $localOutputFolderName = $todayString."_0000";
-
 #back to the destination for the files:
 chdir($todayFolderPath) or handleError("ERROR: Unable to change working directory to output folder path :".$todayFolderPath);
 
@@ -391,7 +413,6 @@ my $sourceFolder = $plotPath."/output/NAMEOUTPUT/".$todayString."0000/";
 my $destFolder = $epiDepFolderName;
 dircopy($sourceFolder, $destFolder) or handleError("ERROR: Unable to copy directory from ".$sourceFolder." to ".$destFolder);
 
-
 #TODO: Bung todays workspace folder onto the RCS