diff --git a/DataProcessor.pl b/DataProcessor.pl index 83f8a83e36261fe7538c2a48be19d533d6c39cb1..bf930e2cbb8223efe4ff1e1e6c2e4326e6a3156e 100755 --- a/DataProcessor.pl +++ b/DataProcessor.pl @@ -4,7 +4,6 @@ use warnings; use strict; use Net::FTP; -use POSIX qw(strftime); use DateTime; use JSON::MaybeXS; use File::Slurp; @@ -14,8 +13,9 @@ use File::Path qw(make_path remove_tree); #TODO: Move elements to json config so that same script can be used to launch both deposition and environmental data processing #Add fixed timestamp generated at the start of call to outputs so possible to distinguish between different runs of the script +my $standardTimeStampFMT = "%Y%m%d%H%M %Z"; sub getTimestampNow { - return strftime("%Y%m%d%H%M", localtime()); + return DateTime->today(time_zone => "UTC")->strftime($standardTimeStampFMT); } my $debugTimeString = getTimestampNow(); @@ -27,7 +27,7 @@ my $debugNoUpload = 0; my $workspacePath = "/storage/app/Ethiopia-EWS/Ethiopia-EWS-Workspace"; my $coordinatorPath = "/storage/app/Ethiopia-EWS/Ethiopia-EWS-Coordinator"; -my $todayString = strftime("%Y%m%d", localtime()); +my $todayString = DateTime->today(time_zone => "UTC")->strftime("%Y%m%d"); my $jobIDString = "DEPOSITION"; @@ -156,8 +156,8 @@ sub handleDataNotAvailableServerDown { my $errMsg = shift; - handleDataNotAvailableForReason($serverUnavailableWarningTime, $jobIDString." processing unable to connect to ".$serverName.": ".$errMsg.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime("%Y%m%d%H%M %Z").". This warning is issued if the server is unreachable after ".$serverUnavailableWarningTime->strftime("%Y%m%d%H%M %Z")." as the ".$jobIDString." data is expected to be available soon"); - handleDataNotAvailableForReason($serverUnavailableErrorTime, $jobIDString." processing unable to connect to ".$serverName.": ".$errMsg.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime("%Y%m%d%H%M %Z").". This warning is issued if the server is unreachable after ".$serverUnavailableErrorTime->strftime("%Y%m%d%H%M %Z"). " as the ".$jobIDString." data is expected to be available already"); + handleDataNotAvailableForReason($serverUnavailableWarningTime, $jobIDString." processing unable to connect to ".$serverName.": ".$errMsg.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime($standardTimeStampFMT).". This warning is issued if the server is unreachable after ".$serverUnavailableWarningTime->strftime($standardTimeStampFMT)." as the ".$jobIDString." data is expected to be available soon"); + handleDataNotAvailableForReason($serverUnavailableErrorTime, $jobIDString." processing unable to connect to ".$serverName.": ".$errMsg.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime($standardTimeStampFMT).". This warning is issued if the server is unreachable after ".$serverUnavailableErrorTime->strftime($standardTimeStampFMT). " as the ".$jobIDString." data is expected to be available already"); handleDataNotAvailableByTime($serverName); #Edge case: The server is down, but it is before the time at which we should issue any warnings - none of the above functions will act, but we still need to terminate this run @@ -168,7 +168,7 @@ sub handleDataNotAvailableByTime { my $serverName = shift; - handleDataNotAvailableForReason($dataNotYetAvailableErrorTime, $jobIDString." data not found on ".$serverName.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime("%Y%m%d%H%M %Z")." this warning is issued if the data is not available after ".$dataNotYetAvailableErrorTime->strftime("%Y%m%d%H%M %Z")); + handleDataNotAvailableForReason($dataNotYetAvailableErrorTime, $jobIDString." data not found on ".$serverName.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime($standardTimeStampFMT)." this warning is issued if the data is not available after ".$dataNotYetAvailableErrorTime->strftime($standardTimeStampFMT)); #This function should not terminate the run itself, the decision is delegated to the above call to handleDataNotAvailableForReason } diff --git a/ENVDataProcessor.pl b/ENVDataProcessor.pl index 3d55c35cda81fd0aafb6558d3fe52c42fe630166..fdd3fc376ec9ad13d63c73aeb8949e8198ca9bce 100755 --- a/ENVDataProcessor.pl +++ b/ENVDataProcessor.pl @@ -4,7 +4,6 @@ use warnings; use strict; use Net::FTP; -use POSIX qw(strftime); use DateTime; use JSON::MaybeXS; use File::Slurp; @@ -14,8 +13,9 @@ use File::Path qw(make_path remove_tree); #TODO: Move elements to json config so that same script can be used to launch both deposition and environmental data processing #Add fixed timestamp generated at the start of call to outputs so possible to distinguish between different runs of the script +my $standardTimeStampFMT = "%Y%m%d%H%M %Z"; sub getTimestampNow { - return strftime("%Y%m%d%H%M", localtime()); + return DateTime->today(time_zone => "UTC")->strftime($standardTimeStampFMT); } my $debugTimeString = getTimestampNow(); @@ -27,7 +27,7 @@ my $debugNoUpload = 0; my $workspacePath = "/storage/app/Ethiopia-EWS/Ethiopia-EWS-Workspace"; my $coordinatorPath = "/storage/app/Ethiopia-EWS/Ethiopia-EWS-Coordinator"; -my $todayString = strftime("%Y%m%d", localtime()); +my $todayString = DateTime->today(time_zone => "UTC")->strftime("%Y%m%d"); my $jobIDString = "ENVIRONMENT"; @@ -39,7 +39,7 @@ my @maintainersMO = ("william.thurston\@metoffice.gov.uk"); my @maintainersAll = (@maintainersCam, @maintainersMO); -my $expectedTimeOfDataAvailibility = DateTime->today(time_zone => "UTC")->add(hours => 8, minutes => 0);#TODO: Deal with the fact that UTC today does not line up with today at midnight when we are on BST so get a warning at 0005 +my $expectedTimeOfDataAvailibility = DateTime->today(time_zone => "UTC")->add(hours => 8, minutes => 0); my $serverUnavailableWarningTime = $expectedTimeOfDataAvailibility->clone()->subtract(minutes => 30); my $serverUnavailableErrorTime = $expectedTimeOfDataAvailibility->clone()->add(minutes => 30); @@ -155,9 +155,9 @@ sub handleDataNotAvailableServerDown { my $serverName = shift; my $errMsg = shift; - #TODO: Did this not send an email when it fired too early? - handleDataNotAvailableForReason($serverUnavailableWarningTime, $jobIDString." processing unable to connect to ".$serverName.": ".$errMsg.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime("%Y%m%d%H%M %Z").". This warning is issued if the server is unreachable after ".$serverUnavailableWarningTime->strftime("%Y%m%d%H%M %Z")." as the ".$jobIDString." data is expected to be available soon"); - handleDataNotAvailableForReason($serverUnavailableErrorTime, $jobIDString." processing unable to connect to ".$serverName.": ".$errMsg.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime("%Y%m%d%H%M %Z").". This warning is issued if the server is unreachable after ".$serverUnavailableErrorTime->strftime("%Y%m%d%H%M %Z"). " as the ".$jobIDString." data is expected to be available already"); + + handleDataNotAvailableForReason($serverUnavailableWarningTime, $jobIDString." processing unable to connect to ".$serverName.": ".$errMsg.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime($standardTimeStampFMT).". This warning is issued if the server is unreachable after ".$serverUnavailableWarningTime->strftime($standardTimeStampFMT)." as the ".$jobIDString." data is expected to be available soon"); + handleDataNotAvailableForReason($serverUnavailableErrorTime, $jobIDString." processing unable to connect to ".$serverName.": ".$errMsg.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime($standardTimeStampFMT).". This warning is issued if the server is unreachable after ".$serverUnavailableErrorTime->strftime($standardTimeStampFMT). " as the ".$jobIDString." data is expected to be available already"); handleDataNotAvailableByTime($serverName); #Edge case: The server is down, but it is before the time at which we should issue any warnings - none of the above functions will act, but we still need to terminate this run @@ -168,7 +168,7 @@ sub handleDataNotAvailableByTime { my $serverName = shift; - handleDataNotAvailableForReason($dataNotYetAvailableErrorTime, $jobIDString." data not found on ".$serverName.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime("%Y%m%d%H%M %Z")." this warning is issued if the data is not available after ".$dataNotYetAvailableErrorTime->strftime("%Y%m%d%H%M %Z")); + handleDataNotAvailableForReason($dataNotYetAvailableErrorTime, $jobIDString." data not found on ".$serverName.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime($standardTimeStampFMT)." this warning is issued if the data is not available after ".$dataNotYetAvailableErrorTime->strftime($standardTimeStampFMT)); #This function should not terminate the run itself, the decision is delegated to the above call to handleDataNotAvailableForReason }