diff --git a/DataProcessor.pl b/DataProcessor.pl index 42c7bff26d98f110f7bbe3f0ff4665d2513bedfa..c1c0a1a1ccaf01d54bf6a7114ac921b26d3d2a3d 100755 --- a/DataProcessor.pl +++ b/DataProcessor.pl @@ -9,6 +9,10 @@ use JSON::MaybeXS; use File::Slurp; use File::Copy::Recursive qw(fcopy rcopy dircopy fmove rmove dirmove); +#TODO: Move elements to json config so that same script can be used to launch both deposition and environmental data processing + +#Add fixed timestamp generated at the start of call to outputs so possible to distinguish between different runs of the script +my $debugTimeString = strftime("%Y%m%d%H%M", localtime()); my $debugOutput = 0; my $workspacePath = "/storage/app/Ethiopia-EWS/Ethiopia-EWS-Workspace"; @@ -30,16 +34,16 @@ sub handleError { #Send email notifications: my @maintainers = ("ca500\@cam.ac.uk", "ds603\@cam.ac.uk", "rs481\@cam.ac.uk"); - @maintainers = ("rs481\@cam.ac.uk");print STDERR "DEBUG: MAINTAINERS LIST\n"; + #@maintainers = ("rs481\@cam.ac.uk");print STDERR "DEBUG: MAINTAINERS LIST\n"; foreach my $maintainer (@maintainers) { - system("/storage/app/servermail/servermail/servermail.sh -r ".$maintainer." -s \"ERROR in the Ethiopia EWS ".$jobIDString." ".$failTimeStamp."\" -m \"".$errorString."\n\nAutomatic running of the ".$jobIDString." has been suspended.\n\nResolve this error and remove the error file ".$failFilePath." to reenable automatic processing of ".$jobIDString."\""); + system("/storage/app/servermail/servermail/servermail.sh -r ".$maintainer." -s \"ERROR in the Ethiopia EWS ".$jobIDString." ".$failTimeStamp."\" -m \"".$errorString."\n\nError encountered at ".$failTimeStamp." during running of ".$jobIDString." initiated at ".$debugTimeString.".\n\nAutomatic running of ".$jobIDString."has been suspended.\n\nResolve this error and remove the error file ".$failFilePath." to reenable automatic processing of ".$jobIDString."\""); } #Log error with error file: - open(my $failFH, ">", $failFilePath) or die "ERROR: Unable to open error file for writing: ".$failFilePath." for error ".$errorString; + open(my $failFH, ">", $failFilePath) or die "ERROR: ".$debugTimeString." Unable to open error file for writing: ".$failFilePath." for error ".$errorString; print $failFH "Failed: ".$failTimeStamp."\n"; print $failFH $errorString."\n"; - close($failFH) or die "ERROR: Unable to write ".$failFilePath." for error ".$errorString; + close($failFH) or die "ERROR: ".$debugTimeString." Unable to write ".$failFilePath." for error ".$errorString; #Remove in-progress flag, how likely to succeed if there has been an error somewhere... if(-e $inProgressFilePath) { @@ -52,7 +56,7 @@ sub handleError { #Transfer has errored out already, needs intervention to fix it: if(-e $failFilePath) { if($debugOutput) { - print("DEBUG: ERROR file (".$failFilePath.") already exists, stopping early pending intervention to resolve issue.\n"); + print("DEBUG: ".$debugTimeString." ERROR file (".$failFilePath.") already exists, stopping early pending intervention to resolve issue.\n"); } exit(); } @@ -66,7 +70,7 @@ unless(-d $todayFolderPath) { if(-e $doneFilePath) { #We have already done the extraction for today: if($debugOutput) { - print("DEBUG: Already done for today\n"); + print("DEBUG: ".$debugTimeString." Already done for today\n"); } exit(); } @@ -74,7 +78,7 @@ if(-e $doneFilePath) { if(-e $inProgressFilePath) { #We already have a job under way: if($debugOutput) { - print("DEBUG: Already a job in progress\n"); + print("DEBUG: ".$debugTimeString." Already a job in progress\n"); } exit(); } else { @@ -90,7 +94,7 @@ chdir($todayFolderPath) or handleError("ERROR: Unable to change working director #credentials stored separately so they aren't in plain text and checked in to the repo: my $serverCredentialsFileName = $workspacePath."/MO_FTP_Cred.json"; my $ftpCredentialsString = read_file($serverCredentialsFileName) or handleError("ERROR: Unable to read FTP credentials from credentials file: ".$serverCredentialsFileName); -my $ftpCredentials = decode_json($ftpCredentialsString) or handleError("ERROR: Unable to parse credentials from JSON in credentials file: ".$serverCredentialsFileName); +my $ftpCredentials = decode_json($ftpCredentialsString) or handleError("ERROR: Unable to parse credentials from JSON in credentials file: ".$serverCredentialsFileName);#Obviously don't ever print the _contents_ of the credentials file to the screen... my $ftpServerName = $ftpCredentials->{"server"}; @@ -114,7 +118,7 @@ unless(-e $todayOutputFileName) { #Not an error, probably just not available yet: #TODO: More prominent logging? Error if not available by a certain time? Explicit check for existence of file on the FTP server so we know difference between failed download and file not present yet? if($debugOutput) { - print("DEBUG: File not available: ".$todayOutputFileName); + print("DEBUG: ".$debugTimeString." File not available: ".$todayOutputFileName); } #Remove the in progress file now that we are going to exit: unlink($inProgressFilePath) or handleError("ERROR: Unable to remove in-progress lock file: ".$inProgressFilePath);