Initial version of automated EWS management. Brings down the NAME deposition data.

DataProcessor.pl

+#!/usr/bin/perl
+
+use warnings;
+use strict;
+
+use Net::FTP;
+use POSIX qw(strftime);
+use JSON::MaybeXS;
+use File::Slurp;
+use File::Copy::Recursive qw(fcopy rcopy dircopy fmove rmove dirmove);
+
+my $debugOutput = 0;
+
+my $workspacePath = "/storage/app/Ethiopia-EWS/Ethiopia-EWS-Workspace";
+
+my $todayString = strftime("%Y%m%d", localtime());
+
+my $todayFolderPath = $workspacePath."/".$todayString;
+
+my $jobIDString = "DEPOSITION";
+
+my $inProgressFilePath = $todayFolderPath."/".$jobIDString."_INPROGRESS.txt";
+my $doneFilePath = $todayFolderPath."/".$jobIDString."_SUCCESS.txt";
+my $failFilePath = $workspacePath."/".$jobIDString."_ERROR.txt";
+
+sub handleError {
+	my $errorString = shift;
+
+	my $failTimeStamp = strftime("%Y%m%d%H%M", localtime());
+
+	#Send email notifications:
+	my @maintainers = ("ca500\@cam.ac.uk", "ds603\@cam.ac.uk", "rs481\@cam.ac.uk");
+	@maintainers = ("rs481\@cam.ac.uk");print STDERR "DEBUG: MAINTAINERS LIST\n";
+	foreach my $maintainer (@maintainers) {
+		system("/storage/app/servermail/servermail/servermail.sh -r ".$maintainer." -s \"ERROR in the Ethiopia EWS ".$jobIDString." ".$failTimeStamp."\" -m \"".$errorString."\n\nAutomatic running of the ".$jobIDString." has been suspended.\n\nResolve this error and remove the error file ".$failFilePath." to reenable automatic processing of ".$jobIDString."\"");
+	}
+
+	#Log error with error file:
+	open(my $failFH, ">", $failFilePath) or die "ERROR: Unable to open error file for writing: ".$failFilePath." for error ".$errorString;
+	print $failFH "Failed: ".$failTimeStamp."\n";
+	print $failFH $errorString."\n";
+	close($failFH) or die "ERROR: Unable to write ".$failFilePath." for error ".$errorString;
+
+	#Remove in-progress flag, how likely to succeed if there has been an error somewhere...
+	if(-e $inProgressFilePath) {
+		unlink($inProgressFilePath)
+	}
+
+	die $errorString;
+}
+
+#Transfer has errored out already, needs intervention to fix it:
+if(-e $failFilePath) {
+	if($debugOutput) {
+		print("DEBUG: ERROR file (".$failFilePath.") already exists, stopping early pending intervention to resolve issue.\n");
+	}
+	exit();
+}
+
+#Make sure our destination exists if not already present:
+unless(-d $todayFolderPath) {
+	mkdir($todayFolderPath) or handleError("ERROR: Unable to create output folder path: ".$todayFolderPath);
+}
+
+#Use the existence of the done file folder as the signal that we have already done the work for today:
+if(-e $doneFilePath) {
+	#We have already done the extraction for today:
+	if($debugOutput) {
+		print("DEBUG: Already done for today\n");
+	}
+	exit();
+}
+
+if(-e $inProgressFilePath) {
+	#We already have a job under way:
+	if($debugOutput) {
+		print("DEBUG: Already a job in progress\n");
+	}
+	exit();
+} else {
+	#Create the in progress file 
+	open(my $progressFH, ">", $inProgressFilePath) or handleError("ERROR: Unable to open ftp transfer confirmation file for writing: ".$inProgressFilePath);
+	print $progressFH "Completed: ".strftime("%Y%m%d%H%M", localtime())."\n";
+	close($progressFH) or handleError("ERROR: Unable to write ".$inProgressFilePath);
+}
+
+#Enter destination for the files:
+chdir($todayFolderPath) or handleError("ERROR: Unable to change working directory to output folder path :".$todayFolderPath);
+
+#credentials stored separately so they aren't in plain text and checked in to the repo:
+my $serverCredentialsFileName = $workspacePath."/MO_FTP_Cred.json";
+my $ftpCredentialsString = read_file($serverCredentialsFileName) or handleError("ERROR: Unable to read FTP credentials from credentials file: ".$serverCredentialsFileName);
+my $ftpCredentials = decode_json($ftpCredentialsString) or handleError("ERROR: Unable to parse credentials from JSON in credentials file: ".$serverCredentialsFileName);
+
+my $ftpServerName = $ftpCredentials->{"server"};
+
+my $ftp = Net::FTP->new($ftpServerName, Debug => 0) or handleError("Cannot connect to ".$ftpServerName.": $@");
+
+$ftp->login($ftpCredentials->{"user"}, $ftpCredentials->{"pass"}) or handleError("Cannot login ", $ftp->message);
+
+$ftp->cwd("/Cambridge_wheatrust_Laura") or handleError("Cannot change working directory ", $ftp->message);
+
+my $todayOutputDir = "NAME_wheatrust_".$todayString;
+my $todayOutputFileName = $todayOutputDir.".tar.gz";
+
+#Get the relevant zipped data file:
+$ftp->get($todayOutputFileName);
+
+$ftp->quit();
+
+
+#Check it came down cleanly:
+unless(-e $todayOutputFileName) {
+	#Not an error, probably just not available yet:
+	#TODO: More prominent logging? Error if not available by a certain time? Explicit check for existence of file on the FTP server so we know difference between failed download and file not present yet?
+	if($debugOutput) {
+		print("DEBUG: File not available: ".$todayOutputFileName);
+	}
+	#Remove the in progress file now that we are going to exit:
+	unlink($inProgressFilePath) or handleError("ERROR: Unable to remove in-progress lock file: ".$inProgressFilePath);
+	exit();
+}
+
+#Extract the zip to a folder:
+system("tar -xzf ".$todayOutputFileName);
+
+#Check we got the folder we were expecting
+unless(-d $todayOutputDir) {
+	handleError("Failure: Didn't manage to unzip the output directory: ".$todayOutputDir);
+}
+
+
+my $plotPath = "/storage/app/Ethiopia-EWS/Plotting/";
+
+#Plotting:
+chdir($plotPath) or handleError("ERROR: Unable to change to plotting directory ".$plotPath);
+
+system("/usr/local/R/bin/Rscript Submit_mass_Gilligan.R config/ds603/NAME_plotting.json --fastanimation --NAME");#Note that these arguments are ordered and it will not work if they are the wrong way round... 
+
+#Plotting outputs in:
+my $plotImagePath = $plotPath."/animation_images/output/NAMEOUTPUT/".$todayString."0000/";
+
+#Bung all necessary outputs into folder named $todayString_0000 here in a nice structure
+my $localOutputFolderName = $todayString."_0000";
+
+#back to the destination for the files:
+chdir($todayFolderPath) or handleError("ERROR: Unable to change working directory to output folder path :".$todayFolderPath);
+
+mkdir($localOutputFolderName) or handleError("ERROR: Unable to create output folder path: ".$localOutputFolderName);
+
+sub moveOutputsToResultFolder {
+	my $resultFolderName = shift;
+	my @outputFolders = @_;
+
+	unless(-d $resultFolderName) {
+		mkdir($resultFolderName) or handleError("ERROR: Unable to create output folder path: ".$resultFolderName);
+	}
+
+	foreach my $outputFolder (@outputFolders) {
+		dircopy($outputFolder, $resultFolderName) or handleError("ERROR: Unable to copy directory from ".$outputFolder." to ".$resultFolderName);
+	}
+}
+
+#Put all the outputs in a subfolder called "All" and put only the daily totals into a subfolder called "Summary"
+my @weeklyPlotOutputFolders = (
+	$plotImagePath."/depositionEthiopia_Stem_total_normalized",
+	$plotImagePath."/depositionEthiopia_Stripe_total_normalized"
+);
+
+moveOutputsToResultFolder($localOutputFolderName."/Weekly", @weeklyPlotOutputFolders);
+
+my @dailyPlotOutputFolders = (
+	$plotImagePath."/depositionEthiopia_Stem_daily_normalized",
+	$plotImagePath."/depositionEthiopia_Stripe_daily_normalized"
+);
+
+moveOutputsToResultFolder($localOutputFolderName."/Daily", @dailyPlotOutputFolders);
+
+my @hourlyPlotOutputFolders = (
+	$plotImagePath."/depositionEthiopia_Stem_normalized",
+	$plotImagePath."/depositionEthiopia_Stripe_normalized"
+);
+
+moveOutputsToResultFolder($localOutputFolderName."/Hourly", @hourlyPlotOutputFolders);
+
+
+#Transfer the output to public server for others to be able to pull it down:
+system("scp -i ".$workspacePath."/ssh_key_willow"." -r ".$localOutputFolderName." willow.csx.cam.ac.uk:/var/www/html/Ethiopia/");
+
+
+
+#Gather parsed output csvs to be used by the EPI model:
+my $epiFolderName = $todayFolderPath."/EPI";
+unless(-d $epiFolderName) {
+	mkdir($epiFolderName) or handleError("ERROR: Unable to create EPI model folder path: ".$epiFolderName);
+}
+
+my $epiDepFolderName = $epiFolderName."/".$jobIDString;
+unless(-d $epiDepFolderName) {
+	mkdir($epiDepFolderName) or handleError("ERROR: Unable to create EPI folder for epi inputs: ".$epiDepFolderName);
+}
+
+my $sourceFolder = $plotPath."/output/NAMEOUTPUT/".$todayString."0000/";
+my $destFolder = $epiDepFolderName;
+dircopy($sourceFolder, $destFolder) or handleError("ERROR: Unable to copy directory from ".$sourceFolder." to ".$destFolder);
+
+
+
+#Successful exit, remove the in progress file:
+unlink($inProgressFilePath) or handleError("ERROR: Unable to remove in-progress lock file: ".$inProgressFilePath);
+
+#Flag overall success of operation:
+open(my $successFH, ">", $doneFilePath) or handleError("ERROR: Unable to open confirmation file for writing: ".$doneFilePath);
+print $successFH "Completed: ".strftime("%Y%m%d%H%M", localtime())."\n";
+close($successFH) or handleError("ERROR: Unable to write ".$doneFilePath);
+
+exit();

NAMEDataCheckerCron.sh

+#!/bin/bash
+
+#Checks to see if todays NAME deposition data is available yet,
+#if so, pulls it down 
+#and starts the portion of the EWS that runs on the Cambridge servers
+
+/storage/app/Ethiopia-EWS/Ethiopia-EWS-Coordinator/DataProcessor.pl >> /storage/app/Ethiopia-EWS/Ethiopia-EWS-Coordinator/NAMEDatalogs.txt 2>&1

NAMEDataCheckerCrontab.txt

+*/5 * * * * /storage/app/Ethiopia-EWS/Ethiopia-EWS-Coordinator/NAMEDataCheckerCron.sh