From 898b447855a124c7dd318e4a84508141d7fb85a7 Mon Sep 17 00:00:00 2001
From: Jake Smith <jws52@cam.ac.uk>
Date: Mon, 4 Jan 2021 16:27:41 +0000
Subject: [PATCH] Cleared out all the perl in this python branch.
---
DataProcessor.pl | 420 ------------------------
ENVDataCheckerCron.sh | 60 ----
ENVDataCheckerCrontab.txt | 1 -
ENVDataProcessor.pl | 434 -------------------------
EnvProcessor.py | 604 -----------------------------------
NAMEDataCheckerCron.sh | 60 ----
NAMEDataCheckerCrontab.txt | 1 -
ProjectConfig_Ethiopia.pl | 35 --
ProjectConfig_SouthAsia.pl | 46 ---
SurveyDataCheckerCrontab.txt | 2 -
SurveyDataProcessor.pl | 582 ---------------------------------
TEST_NAME_Processing.pl | 65 ----
testClearTodayData.pl | 166 ----------
13 files changed, 2476 deletions(-)
delete mode 100755 DataProcessor.pl
delete mode 100755 ENVDataCheckerCron.sh
delete mode 100644 ENVDataCheckerCrontab.txt
delete mode 100755 ENVDataProcessor.pl
delete mode 100644 EnvProcessor.py
delete mode 100755 NAMEDataCheckerCron.sh
delete mode 100644 NAMEDataCheckerCrontab.txt
delete mode 100644 ProjectConfig_Ethiopia.pl
delete mode 100644 ProjectConfig_SouthAsia.pl
delete mode 100644 SurveyDataCheckerCrontab.txt
delete mode 100755 SurveyDataProcessor.pl
delete mode 100755 TEST_NAME_Processing.pl
delete mode 100755 testClearTodayData.pl
diff --git a/DataProcessor.pl b/DataProcessor.pl
deleted file mode 100755
index 70522ad..0000000
--- a/DataProcessor.pl
+++ /dev/null
@@ -1,420 +0,0 @@
-#!/usr/bin/perl
-
-use warnings;
-use strict;
-
-use Net::FTP;
-use DateTime;
-use JSON::MaybeXS;
-use File::Slurp;
-use File::Copy::Recursive qw(fcopy rcopy dircopy fmove rmove dirmove);
-use File::Path qw(make_path remove_tree);
-use File::Basename;
-
-#TODO: Move elements to json config so that same script can be used to launch both deposition and environmental data processing
-
-#Add fixed timestamp generated at the start of call to outputs so possible to distinguish between different runs of the script
-my $standardTimeStampFMT = "%Y%m%d%H%M %Z";
-sub getTimestampNow {
- return DateTime->now(time_zone => "UTC")->strftime($standardTimeStampFMT);
-}
-my $debugTimeString = getTimestampNow();
-
-my $debugOutput = 1;
-
-my $debugNoFTPDownload = 0;
-my $debugNoUpload = 0;
-my $debugNoPlot = 0;
-
-# read in project configuration from input argument
-my $configFilePath = shift;
-require $configFilePath;
-
-# get variables saved in project configuration
-my $projectName = get_project_name();
-my $projectPath = get_project_path();
-
-# workspace is in project directory
-my $workspacePath = $projectPath."Workspace";
-# coordinator space is generalised
-my $coordinatorPath = dirname($configFilePath);
-
-my $downloadFromWillow = get_download_from_willow();
-
-my $todayString = shift;
-
-# if date isn't an input argument, use today's date for operational
-if (not defined $todayString) {
- if($debugOutput) {
- print("DEBUG: ".$debugTimeString." Date not provided as input argument, so using today's date.\n");
- }
- $todayString = DateTime->today(time_zone => "UTC")->strftime("%Y%m%d");
-}
-
-my $jobIDString = "DEPOSITION";
-
-my $todayFolderPath = $workspacePath."/DEPOSITION_".$todayString;
-
-my $NAMEDirectoryPrefix = get_NAME_directory_prefix();
-my @NAMEFilePrefixes = get_NAME_file_prefixes();
-
-my @maintainersCam = ("jws52\@cam.ac.uk", "tm689\@cam.ac.uk", "rs481\@cam.ac.uk");
-my @maintainersMO = ("william.thurston\@metoffice.gov.uk");
-my @maintainersAll = (@maintainersCam, @maintainersMO);
-
-my $expectDataAvailableHour = get_expected_available_hour();
-my $expectDataAvailableMinute = get_expected_available_minute();
-my $expectedTimeOfDataAvailibility = DateTime->today(time_zone => "UTC")->add(hours => $expectDataAvailableHour, minutes => $expectDataAvailableMinute);
-
-my $serverUnavailableWarningTime = $expectedTimeOfDataAvailibility->clone()->subtract(minutes => 30);
-my $serverUnavailableErrorTime = $expectedTimeOfDataAvailibility->clone()->add(minutes => 30);
-
-my $dataNotYetAvailableErrorTime = $expectedTimeOfDataAvailibility->clone()->add(minutes => 195);
-
-
-my $inProgressFilePath = $todayFolderPath."/".$jobIDString."_INPROGRESS.txt";
-my $doneFilePath = $todayFolderPath."/".$jobIDString."_SUCCESS.txt";
-
-my $severityStringERROR = "ERROR";
-my $severityStringWARNING = "WARNING";
-
-my $failFilePath = $todayFolderPath."/".$jobIDString."_ERROR.txt";
-my $warningFilePath = $todayFolderPath."/".$jobIDString."_WARNING.txt";
-
-
-sub beginRun {
- #Create the in progress file
- open(my $progressFH, ">", $inProgressFilePath) or handleError("ERROR: Unable to open ".$jobIDString." in-progress file for writing: ".$inProgressFilePath);
- print $progressFH "Started: ".getTimestampNow()."\n";
- close($progressFH) or handleError("ERROR: Unable to write ".$inProgressFilePath);
-
- # initiate log
- print("--------------\n");
- print($debugTimeString."\n");
- print("DataProcessor.pl in progress!\n")
-}
-
-sub terminateRun {
- if(-e $inProgressFilePath) {
- unlink($inProgressFilePath) or handleError("ERROR: Unable to remove in-progress lock file: ".$inProgressFilePath);
- }
- exit();
-}
-
-sub handleError {
- my $errorString = shift;
-
- my $maintainerRef = shift // \@maintainersCam;
-
- my $severity = shift // $severityStringERROR;
-
- my $severityString = $severityStringERROR;
- my $severityFile = $failFilePath;
- #Treat anything but a correctly specified warning as an error
- if($severity eq $severityStringWARNING) {
- $severityString = $severityStringWARNING;
- $severityFile = $warningFilePath;
- }
-
- my $failTimeStamp = getTimestampNow();
-
- #Send email notifications:
- my $emailArgs = " -s \"".$severityString." in the ".$projectName." EWS ".$jobIDString." ".$failTimeStamp."\" -m \"".$errorString."\n\n".$severityString." encountered at ".$failTimeStamp." during running of ".$jobIDString." initiated at ".$debugTimeString.".";
- if($severityString eq $severityStringERROR) {
- $emailArgs .= "\n\nAutomatic running of ".$jobIDString." has been suspended.\n\nResolve this error and remove the error file ".$failFilePath." to reenable automatic processing of ".$jobIDString;
- }
- $emailArgs .= "\"";
-
- print "Messaging maintainers:\n";
- foreach my $maintainer (@$maintainerRef) {
- print "Messaging maintainer: ".$maintainer."\n";
- system("/storage/app/servermail/servermail/servermail.sh -r ".$maintainer." ".$emailArgs);
- }
- #TODO: How do we tell if the servermail call failed? Adapt servermail scrip to leave an error file (or return code?) and then look for this.
- #TODO: If servermail fails, wait a bit (5 minutes?) and then try to send mail again (5 times?). If all those fail, try a different method of communication.
- #TODO: Set up and outside service to see if pine and willow get cut off from outside world
-
- #Log error with error file:
- open(my $failFH, ">>", $severityFile) or die "ERROR: ".$debugTimeString." Unable to open error file for writing: ".$severityFile." for error ".$errorString;
- print $failFH "Failed: ".$failTimeStamp."\n";
- print $failFH $errorString."\n";
- close($failFH) or die "ERROR: ".$debugTimeString." Unable to write ".$severityFile." for error ".$errorString;
-
- #Remove in-progress flag file as this job is dying, but don't use terminateRun as we are going to die rather than clean exit:
- if(-e $inProgressFilePath) {
- unlink($inProgressFilePath)
- }
-
- die $errorString;
-}
-
-sub handleDataNotAvailableForReason {
-
- my $expectedAvailableTime = shift;
-
- my $warningString = shift;
-
- my $tNow = DateTime->now(time_zone => 'UTC');
-
- if($tNow < $expectedAvailableTime) {
- return;
- }
-
- if(-e $warningFilePath) {
- #if warning file already contains a warning for this reason, don't warn again
- my $alreadyWarned = 0;
- open(my $warningFH, "<", $warningFilePath) or handleError("ERROR: Unable to open ".$jobIDString." warning file for reading: ".$warningFilePath);
- while(my $warnString = <$warningFH>) {
- if($warnString =~ /$warningString/) {
- $alreadyWarned = 1;
- last;
- }
- }
- close($warningFH) or handleError("ERROR: Unable to close ".$warningFilePath." after reading");
- if($alreadyWarned == 1) {
- return;
- }
- }
-
- #We haven't warned for this reason yet, so warn people:
- handleError($warningString, \@maintainersAll, $severityStringWARNING);
-}
-
-sub handleDataNotAvailableServerDown {
-
- my $serverName = shift;
-
- my $errMsg = shift;
-
- handleDataNotAvailableForReason($serverUnavailableWarningTime, $jobIDString." processing unable to connect to ".$serverName.": ".$errMsg.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime($standardTimeStampFMT).". This warning is issued if the server is unreachable after ".$serverUnavailableWarningTime->strftime($standardTimeStampFMT)." as the ".$jobIDString." data is expected to be available soon");
- handleDataNotAvailableForReason($serverUnavailableErrorTime, $jobIDString." processing unable to connect to ".$serverName.": ".$errMsg.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime($standardTimeStampFMT).". This warning is issued if the server is unreachable after ".$serverUnavailableErrorTime->strftime($standardTimeStampFMT). " as the ".$jobIDString." data is expected to be available already");
- handleDataNotAvailableByTime($serverName);
-
- #Edge case: The server is down, but it is before the time at which we should issue any warnings - none of the above functions will act, but we still need to terminate this run
- terminateRun();
-}
-
-sub handleDataNotAvailableByTime {
-
- my $serverName = shift;
-
- handleDataNotAvailableForReason($dataNotYetAvailableErrorTime, $jobIDString." data not found on ".$serverName.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime($standardTimeStampFMT)." this warning is issued if the data is not available after ".$dataNotYetAvailableErrorTime->strftime($standardTimeStampFMT));
-
- #This function should not terminate the run itself, the decision is delegated to the above call to handleDataNotAvailableForReason
-}
-
-
-#Transfer has errored out already, needs intervention to fix it:
-if(-e $failFilePath) {
- if($debugOutput) {
- print("DEBUG: ".$debugTimeString." ERROR file (".$failFilePath.") already exists, stopping early pending intervention to resolve issue.\n");
- }
- terminateRun();
-}
-
-#Make sure our destination exists if not already present:
-unless(-d $todayFolderPath) {
- mkdir($todayFolderPath) or handleError("ERROR: Unable to create output folder path: ".$todayFolderPath);
-}
-
-#Use the existence of the done file folder as the signal that we have already done the work for today:
-if(-e $doneFilePath) {
- #We have already done the extraction for today:
- if($debugOutput) {
- print("DEBUG: ".$debugTimeString." Already done for today\n");
- }
- terminateRun();
-}
-
-if(-e $inProgressFilePath) {
- #We already have a job under way:
- if($debugOutput) {
- print("DEBUG: ".$debugTimeString." Already a job in progress\n");
- }
- #NOTE: Don't call terminateRun here, as we must not delete the in progress file of the other job
- exit();
-}
-
-#Clear for this job to start its run:
-beginRun();
-
-#Enter destination for the files:
-chdir($todayFolderPath) or handleError("ERROR: Unable to change working directory to output folder path :".$todayFolderPath);
-
-my $todayOutputDir = $NAMEDirectoryPrefix.$todayString;
-my $todayOutputFileName = $todayOutputDir.".tar.gz";
-
-unless($debugNoFTPDownload) {
-
- # this variable wants to be named outside fo the below blocks
- my $ftpServerName = "willow.csx.cam.ac.uk";
-
- if ($downloadFromWillow) {
-
- # TODO: make a file check and wait 20s before downloading, in case scp is called before file is completely available
- # If file is not there, script will not die, instead it prints warning to logfile
- my $SCPCommand = "scp -i ".$coordinatorPath."/ssh_key_willow willow.csx.cam.ac.uk:/storage/sftp/metofficeupload/upload/".$projectName."/fromMO/daily_name/".$todayOutputFileName." ".$todayFolderPath."/";
-
- # system returns 1 on failure, so perl picks this up with 'and'
- system($SCPCommand) and handleDataNotAvailableByTime($ftpServerName);
-
- } else {
- #credentials stored separately so they aren't in plain text and checked in to the repo:
- my $serverCredentialsFileName = $coordinatorPath."/MO_FTP_Cred.json";
- my $ftpCredentialsString = read_file($serverCredentialsFileName) or handleError("ERROR: Unable to read FTP credentials from credentials file: ".$serverCredentialsFileName);
- my $ftpCredentials = decode_json($ftpCredentialsString) or handleError("ERROR: Unable to parse credentials from JSON in credentials file: ".$serverCredentialsFileName);#Obviously don't ever print the _contents_ of the credentials file to the screen...
-
- $ftpServerName = $ftpCredentials->{"server"};
-
- #We handle the FTP being down, or we will keep retrying until it comes back up, but we will warn the MO if it is not up around the time the data is expected to be available
- my $ftp = Net::FTP->new($ftpServerName, Debug => 0) or handleDataNotAvailableServerDown($ftpServerName, $@);
-
- $ftp->login($ftpCredentials->{"user"}, $ftpCredentials->{"pass"}) or handleError("Cannot login ", $ftp->message);#NOTE: This error would be their fault, but we need to communicate properly to fix it
-
- $ftp->cwd("/Cambridge_wheatrust_Laura") or handleError("Cannot change working directory ", $ftp->message);#NOTE: This error would be their fault, but we need to communicate properly to fix it
-
- #Get the relevant zipped data file:
- $ftp->get($todayOutputFileName);
-
- $ftp->quit();
- }
-
- #Check it came down cleanly:
- unless(-e $todayOutputFileName) {
- #Not necessarily an error, probably just not available yet:
- if($debugOutput) {
- print("DEBUG: ".$debugTimeString." File not available: ".$todayOutputFileName);
- }
- #TODO: More prominent logging? Error if not available by a certain time? Explicit check for existence of file on the FTP server so we know difference between failed download and file not present yet?
- #TODO: TODAY Should email met office (+Cam team) if files are not present by $dataNotYetAvailableErrorTime (and make sure don't email multiple times...)
- handleDataNotAvailableByTime($ftpServerName);
-
- #Whether or not the data should have been here by now, we're done:
- terminateRun();
- }
-
- #Extract the zip to a folder:
- system("tar -xzf ".$todayOutputFileName);
-}
-
-#Check we got the folder we were expecting
-unless(-d $todayOutputDir) {
- handleError("ERROR: Didn't manage to unzip the output directory: ".$todayOutputDir." from ".$todayOutputFileName, \@maintainersAll);#Email the met office as well as the error is on their end -either totally empty zip or misnamed file
-}
-
-#Rename the folder for Dan:
-my $oldTodayOutputDir = $todayOutputDir;
-$todayOutputDir = $NAMEDirectoryPrefix.$todayString;
-
-unless($oldTodayOutputDir eq $todayOutputDir) {#They will only differ if we're using a set of testing NAME output
- dircopy($oldTodayOutputDir, $todayOutputDir) or handleError("ERROR: Unable to copy directory from ".$oldTodayOutputDir." to ".$todayOutputDir);
-}
-
-
-#Check if the zip actually had output inside it - maininput.txt and at least one file of each depositionEthiopia_Stem_*.txt, depositionEthiopia_Stripe_*.txt
-my @necessaryOutputs = ($todayOutputDir."/maininput.txt");
-foreach my $NAMEFilePrefix (@NAMEFilePrefixes) {
- push @necessaryOutputs, $todayOutputDir."/".$NAMEFilePrefix."C1_T1_".$todayString."0300.txt";
-}
-
-foreach my $necessaryOutput (@necessaryOutputs) {
- unless(-e $necessaryOutput) {
- handleError("ERROR: output directory ".$todayOutputDir." unzipped from ".$todayOutputFileName." did not contain required file ".$necessaryOutput, \@maintainersAll);#Email the met office as well as the error is on their end - missing actual output files
- }
-}
-my $plotPath = $todayFolderPath."/plotting/";
-
-#Bung all necessary outputs into folder named $todayString_0000 here in a nice structure
-my $localOutputFolderName = $todayString."_0000";
-
-#Plotting:
-if($debugNoPlot) {
- #handleError("No plotting taking place", \@maintainersCam, $severityStringWARNING);
- if($debugOutput) {
- print("DEBUG: No plotting taking place");
- }
-
- if($debugNoUpload) {
- # no more work to do, so clear up in-progress file
-
- #Successful run, no plotting expected at beginning of Feb 2020.
- # placeholder file state this
- my $noplotFilePath = $todayFolderPath."/".$jobIDString."_NO_PLOTTING.txt";
- open(my $noplotFH, ">", $noplotFilePath) or handleError("ERROR: Unable to open confirmation file for writing: ".$doneFilePath);
- print $noplotFH "Completed: ".getTimestampNow()."\n";
- close($noplotFH) or handleError("ERROR: Unable to write ".$noplotFilePath);
-
- #Flag overall success of operation:
- open(my $successFH, ">", $doneFilePath) or handleError("ERROR: Unable to open confirmation file for writing: ".$doneFilePath);
- print $successFH "Completed: ".getTimestampNow()."\n";
- close($successFH) or handleError("ERROR: Unable to write ".$doneFilePath);
-
- #Remove the in progress file and exit:
- terminateRun();
- }
-}
-
-unless($debugNoPlot) {
- # prepare for new plotting routine
- my @plottingCommands = get_dep_plotting_cmds();
-
- foreach my $plotCommand (@plottingCommands) {
-
- # FALSE option so that hourly plots are not produced on first pass (to save time)
- #TODO: Handle 3hourly images later if needed
- my $plotOptions = $todayString." FALSE";
-
- # run the plot command
- system($plotCommand.' '.$plotOptions);
-
- # check there is some output
- unless(-d $plotPath) { handleError("ERROR: No plotting output produced") };
- }
-}
-
-#Plotting outputs in:
-my @plotPaths = glob($plotPath."{bangladesh,nepal}/images/{Daily,Weekly}");
-#my @plotPaths = glob($plotPath."{bangladesh,nepal}/images/{Daily,Weekly,Hourly}");
-
-#Transfer the output to public server for others to be able to pull it down:
-unless($debugNoUpload) {
-
- # command to make remote directory
- my $willowDir1 = "/var/www/html/".$projectName."/".$localOutputFolderName;
- my $mkdir_cmd = "ssh -i ".$coordinatorPath."/ssh_key_willow willow.csx.cam.ac.uk \"mkdir -p ".$willowDir1."\"";
- print($mkdir_cmd."\n");
- system($mkdir_cmd) and handleError("ERROR: Failed to create output directory on willow");
-
- # command to copy to remote directory
- foreach my $plotPathi (@plotPaths) {
- my $scp_cmd = "scp -i ".$coordinatorPath."/ssh_key_willow -r ".$plotPathi." willow.csx.cam.ac.uk:".$willowDir1."/";
- print($scp_cmd."\n");
- system($scp_cmd) and handleError("ERROR: Failed to copy plotting output to willow");
- }
-
- # Also transfer to sftp directory for others to automatically pull it down
- my $willowDir2 = "/storage/sftp/ewsguest/from-cam/".$projectName."/".$localOutputFolderName;
- $mkdir_cmd = "ssh -i ".$coordinatorPath."/ssh_key_willow willow.csx.cam.ac.uk \"mkdir -p ".$willowDir2."\"";
- print($mkdir_cmd."\n");
- system($mkdir_cmd) and handleError("ERROR: Failed to create output directory on willow");
-
- # command to copy to remote directory
- foreach my $plotPathi (@plotPaths) {
- my $scp_cmd = "scp -i ".$coordinatorPath."/ssh_key_willow -r ".$plotPathi." willow.csx.cam.ac.uk:".$willowDir2."/";
- print($scp_cmd."\n");
- system($scp_cmd) and handleError("ERROR: Failed to copy plotting output to willow");
- }
-}
-
-#No longer a need to gather parsed output csvs to be used by the EPI model, because EWS-plotting does this
-# look in $plotPath/plotting/*/input_csvs/
-
-#TODO: Bung todays workspace folder onto the RCS
-
-#Flag overall success of operation:
-open(my $successFH, ">", $doneFilePath) or handleError("ERROR: Unable to open confirmation file for writing: ".$doneFilePath);
-print $successFH "Completed: ".getTimestampNow()."\n";
-close($successFH) or handleError("ERROR: Unable to write ".$doneFilePath);
-
-#Successful run, remove the in progress file and exit:
-terminateRun();
diff --git a/ENVDataCheckerCron.sh b/ENVDataCheckerCron.sh
deleted file mode 100755
index 11f6cdd..0000000
--- a/ENVDataCheckerCron.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/bin/bash
-
-# Get paths
-coordpath="/storage/app/EWS/General/EWS-Coordinator/"
-
-configpath="$1"
-
-todaydate="$2"
-
-projectpath="$(perl -MPOSIX -e 'require $ARGV[0]; print get_project_path();' -- "$configpath")"
-
-# Start log and check paths exist
-logpath=${projectpath}Workspace/logs/
-logfile=${logpath}ENVDatalogs.txt
-
-if [[ ! -d ${logpath} ]]; then
- printf "Logfile directory does not exist"
- exit 1
-fi
-
-logstr1="=============\n"
-logstr1+="$( date ) \n"
-logstr1+="Looking for coordinator scripts in "${coordpath}"\n"
-logstr1+="Project configuration file is "${configpath}"\n"
-logstr1+="Project directory is "${projectpath}"\n"
-
-if [[ ! -d ${coordpath} ]]; then
- logstr1+="Coordinator directory does not exist"
- printf "${logstr1}" >> ${logfile}
- exit 1
-elif [[ ! -f ${configpath} ]]; then
- logstr1+="Configuration file does not exist"
- printf "${logstr1}" >> ${logfile}
- exit 1
-elif [[ ! -d ${projectpath} ]]; then
- logstr1+="Project directory does not exist"
- printf "${logstr1}" >> ${logfile}
- exit 1
-fi
-
-printf "${logstr1}" >> ${logfile}
-
-#Checks to see if todays ENVIRONMENT data is available yet,
-#if so, pulls it down
-#and starts the portion of the EWS that runs on the Cambridge servers
-${coordpath}ENVDataProcessor.pl ${configpath} ${todaydate} >> ${logfile} 2>&1
-
-# check if perl script ran cleanly, 0 means success
-if [ $? -ne 0 ]; then
- logstr2="ENVDataProcessor.pl failed to finish cleanly.\n"
- logstr2+="=============\n"
- printf "${logstr2}" >> ${logfile}
- exit 1
-fi
-
-logstr2="\n"
-logstr2+="ENVDataProcessor.pl finished cleanly.\n"
-logstr2+="=============\n"
-printf "${logstr2}" >> ${logfile}
-exit 0
diff --git a/ENVDataCheckerCrontab.txt b/ENVDataCheckerCrontab.txt
deleted file mode 100644
index 5d56688..0000000
--- a/ENVDataCheckerCrontab.txt
+++ /dev/null
@@ -1 +0,0 @@
-*/5 * * * * /storage/app/EWS/General/EWS-Coordinator/ENVDataCheckerCron.sh /storage/app/EWS/General/EWS-Coordinator/ProjectConfig_SouthAsia.pl
diff --git a/ENVDataProcessor.pl b/ENVDataProcessor.pl
deleted file mode 100755
index f4beb3b..0000000
--- a/ENVDataProcessor.pl
+++ /dev/null
@@ -1,434 +0,0 @@
-#!/usr/bin/perl
-
-use warnings;
-use strict;
-
-use Net::FTP;
-use DateTime;
-use JSON::MaybeXS;
-use File::Slurp;
-use File::Copy::Recursive qw(fcopy rcopy dircopy fmove rmove dirmove);
-use File::Path qw(make_path remove_tree);
-use File::Basename;
-
-#TODO: Move elements to json config so that same script can be used to launch both deposition and environmental data processing
-
-#Add fixed timestamp generated at the start of call to outputs so possible to distinguish between different runs of the script
-my $standardTimeStampFMT = "%Y%m%d%H%M %Z";
-sub getTimestampNow {
- return DateTime->now(time_zone => "UTC")->strftime($standardTimeStampFMT);
-}
-my $debugTimeString = getTimestampNow();
-
-my $debugOutput = 1;
-
-my $debugNoFTPDownload = 0;
-my $debugNoUpload = 0;
-my $debugNoPlot = 0;
-
-# read in project configuration from input argument
-my $configFilePath = shift;
-require $configFilePath;
-
-#Get variables saved in project configuration
-my $projectName = get_project_name();
-my $projectPath = get_project_path();
-
-# workspace is in project directory
-my $workspacePath = $projectPath."/Workspace";
-# coordinator space is generalised
-my $coordinatorPath = dirname($configFilePath);
-
-my $downloadFromWillow = get_download_from_willow();
-
-# get today's date from second command line argument
-# (if not provided it will be undefined here)
-my $todayString = shift;
-
-# if date isn't an input argument, use today's date for operational
-if (not defined $todayString) {
- if($debugOutput) {
- print("DEBUG: ".$debugTimeString." Date not provided as input argument, so using today's date.\n");
- }
- $todayString = DateTime->today(time_zone => "UTC")->strftime("%Y%m%d");
-} else {
- if($debugOutput) {
- print("DEBUG: ".$debugTimeString." Date provided as input argument, working on ".$todayString."\n");
- }
-}
-
-my $jobIDString = "ENVIRONMENT";
-
-my $todayFolderPath = $workspacePath."/".$jobIDString."_".$todayString;
-
-my $ENVDirectoryPrefix = get_ENV_directory_prefix();
-
-my @maintainersCam = get_maintainers_cam();
-my @maintainersAll = get_maintainers_all();
-
-my $expectDataAvailableHour = get_expected_available_hour();
-my $expectDataAvailableMinute = get_expected_available_minute();
-my $expectedTimeOfDataAvailibility = DateTime->today(time_zone => "UTC")->add(hours => $expectDataAvailableHour, minutes => $expectDataAvailableMinute);
-
-my $serverUnavailableWarningTime = $expectedTimeOfDataAvailibility->clone()->subtract(minutes => 30);
-my $serverUnavailableErrorTime = $expectedTimeOfDataAvailibility->clone()->add(minutes => 30);
-
-my $dataNotYetAvailableErrorTime = $expectedTimeOfDataAvailibility->clone()->add(minutes => 120);
-
-
-my $inProgressFilePath = $todayFolderPath."/".$jobIDString."_INPROGRESS.txt";
-my $doneFilePath = $todayFolderPath."/".$jobIDString."_SUCCESS.txt";
-
-my $severityStringERROR = "ERROR";
-my $severityStringWARNING = "WARNING";
-
-my $failFilePath = $todayFolderPath."/".$jobIDString."_ERROR.txt";
-my $warningFilePath = $todayFolderPath."/".$jobIDString."_WARNING.txt";
-
-
-sub beginRun {
- #Create the in progress file
- open(my $progressFH, ">", $inProgressFilePath) or handleError("ERROR: Unable to open ".$jobIDString." in-progress file for writing: ".$inProgressFilePath);
- print $progressFH "Started: ".getTimestampNow()."\n";
- close($progressFH) or handleError("ERROR: Unable to write ".$inProgressFilePath);
-
- # initiate log
- print("--------------\n");
- print($debugTimeString."\n");
- print("ENVProcessor.pl in progress!\n")
-}
-
-sub terminateRun {
- if(-e $inProgressFilePath) {
- unlink($inProgressFilePath) or handleError("ERROR: Unable to remove in-progress lock file: ".$inProgressFilePath);
- }
- exit();
-}
-
-sub handleError {
- my $errorString = shift;
-
- my $maintainerRef = shift // \@maintainersCam;
-
- my $severity = shift // $severityStringERROR;
-
- my $severityString = $severityStringERROR;
- my $severityFile = $failFilePath;
- #Treat anything but a correctly specified warning as an error
- if($severity eq $severityStringWARNING) {
- $severityString = $severityStringWARNING;
- $severityFile = $warningFilePath;
- }
-
- my $failTimeStamp = getTimestampNow();
-
- #Send email notifications:
- my $emailArgs = " -s \"".$severityString." in the ".$projectName." EWS ".$jobIDString." ".$failTimeStamp."\" -m \"".$errorString."\n\n".$severityString." encountered at ".$failTimeStamp." during running of ".$jobIDString." initiated at ".$debugTimeString.".";
- if($severityString eq $severityStringERROR) {
- $emailArgs .= "\n\nAutomatic running of ".$jobIDString." has been suspended.\n\nResolve this error and remove the error file ".$failFilePath." to reenable automatic processing of ".$jobIDString;
- }
- $emailArgs .= "\"";
-
- print "Messaging maintainers:\n";
- foreach my $maintainer (@$maintainerRef) {
- print "Messaging maintainer: ".$maintainer."\n";
- system("/storage/app/servermail/servermail/servermail.sh -r ".$maintainer." ".$emailArgs);
- }
- #TODO: How do we tell if the servermail call failed? Adapt servermail scrip to leave an error file (or return code?) and then look for this.
- #TODO: If servermail fails, wait a bit (5 minutes?) and then try to send mail again (5 times?). If all those fail, try a different method of communication.
- #TODO: Set up and outside service to see if pine and willow get cut off from outside world
-
- #Log error with error file:
- open(my $failFH, ">>", $severityFile) or die "ERROR: ".$debugTimeString." Unable to open error file for writing: ".$severityFile." for error ".$errorString;
- print $failFH "Failed: ".$failTimeStamp."\n";
- print $failFH $errorString."\n";
- close($failFH) or die "ERROR: ".$debugTimeString." Unable to write ".$severityFile." for error ".$errorString;
-
- #Remove in-progress flag file as this job is dying, but don't use terminateRun as we are going to die rather than clean exit:
- if(-e $inProgressFilePath) {
- unlink($inProgressFilePath)
- }
-
- die $errorString;
-}
-
-sub handleDataNotAvailableForReason {
-
- my $expectedAvailableTime = shift;
-
- my $warningString = shift;
-
- my $tNow = DateTime->now(time_zone => 'UTC');
-
- if($tNow < $expectedAvailableTime) {
- return;
- }
-
- if(-e $warningFilePath) {
- #if warning file already contains a warning for this reason, don't warn again
- my $alreadyWarned = 0;
- open(my $warningFH, "<", $warningFilePath) or handleError("ERROR: Unable to open ".$jobIDString." warning file for reading: ".$warningFilePath);
- while(my $warnString = <$warningFH>) {
- if($warnString =~ /$warningString/) {
- $alreadyWarned = 1;
- last;
- }
- }
- close($warningFH) or handleError("ERROR: Unable to close ".$warningFilePath." after reading");
- if($alreadyWarned == 1) {
- return;
- }
- }
-
- #We haven't warned for this reason yet, so warn people:
- handleError($warningString, \@maintainersAll, $severityStringWARNING);
-}
-
-sub handleDataNotAvailableServerDown {
-
- my $serverName = shift;
-
- my $errMsg = shift;
-
- handleDataNotAvailableForReason($serverUnavailableWarningTime, $jobIDString." processing unable to connect to ".$serverName.": ".$errMsg.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime($standardTimeStampFMT).". This warning is issued if the server is unreachable after ".$serverUnavailableWarningTime->strftime($standardTimeStampFMT)." as the ".$jobIDString." data is expected to be available soon");
- handleDataNotAvailableForReason($serverUnavailableErrorTime, $jobIDString." processing unable to connect to ".$serverName.": ".$errMsg.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime($standardTimeStampFMT).". This warning is issued if the server is unreachable after ".$serverUnavailableErrorTime->strftime($standardTimeStampFMT). " as the ".$jobIDString." data is expected to be available already");
- handleDataNotAvailableByTime($serverName);
-
- #Edge case: The server is down, but it is before the time at which we should issue any warnings - none of the above functions will act, but we still need to terminate this run
- terminateRun();
-}
-
-sub handleDataNotAvailableByTime {
-
- my $serverName = shift;
-
- handleDataNotAvailableForReason($dataNotYetAvailableErrorTime, $jobIDString." data not found on ".$serverName.". Data expected to be available at ".$expectedTimeOfDataAvailibility->strftime($standardTimeStampFMT)." this warning is issued if the data is not available after ".$dataNotYetAvailableErrorTime->strftime($standardTimeStampFMT));
-
- #This function should not terminate the run itself, the decision is delegated to the above call to handleDataNotAvailableForReason
-}
-
-
-#Transfer has errored out already, needs intervention to fix it:
-if(-e $failFilePath) {
- if($debugOutput) {
- print("DEBUG: ".$debugTimeString." ERROR file (".$failFilePath.") already exists, stopping early pending intervention to resolve issue.\n");
- }
- terminateRun();
-}
-
-#Make sure our destination exists if not already present:
-unless(-d $todayFolderPath) {
- mkdir($todayFolderPath) or handleError("ERROR: Unable to create output folder path: ".$todayFolderPath);
-}
-
-#Use the existence of the done file folder as the signal that we have already done the work for today:
-if(-e $doneFilePath) {
- #We have already done the extraction for today:
- if($debugOutput) {
- print("DEBUG: ".$debugTimeString." Already done for today\n");
- }
- terminateRun();
-}
-
-if(-e $inProgressFilePath) {
- #We already have a job under way:
- if($debugOutput) {
- print("DEBUG: ".$debugTimeString." Already a job in progress\n");
- }
- #NOTE: Don't call terminateRun here, as we must not delete the in progress file of the other job
- exit();
-}
-
-#Clear for this job to start its run:
-beginRun();
-
-#Enter destination for the files:
-chdir($todayFolderPath) or handleError("ERROR: Unable to change working directory to output folder path :".$todayFolderPath);
-
-my $todayOutputDir = $ENVDirectoryPrefix.$todayString;
-my $todayOutputFileName = $todayOutputDir.".tar.gz";
-
-unless($debugNoFTPDownload) {
-
- # this variable wants to be named outside fo the below blocks
- my $ftpServerName = "willow.csx.cam.ac.uk";
-
- if ($downloadFromWillow) {
- # get from willow sftp
-
- # TODO: make a file check and wait 20s before downloading, in case scp is called before file is completely available
- # If file is not there, script will not die, instead it prints warning to logfile
- my $SCPCommand = "scp -i ".$coordinatorPath."/ssh_key_willow willow.csx.cam.ac.uk:/storage/sftp/metofficeupload/upload/".$projectName."/fromMO/daily_name/".$todayOutputFileName." ".$todayFolderPath."/";
-
- # system returns 1 on failure, so perl picks this up with 'and'
- system($SCPCommand) and handleDataNotAvailableByTime($ftpServerName);
-
- } else {
- # get from met office ftp server
-
- #credentials stored separately so they aren't in plain text and checked in to the repo:
- my $serverCredentialsFileName = $coordinatorPath."/MO_FTP_Cred.json";
- my $ftpCredentialsString = read_file($serverCredentialsFileName) or handleError("ERROR: Unable to read FTP credentials from credentials file: ".$serverCredentialsFileName);
- my $ftpCredentials = decode_json($ftpCredentialsString) or handleError("ERROR: Unable to parse credentials from JSON in credentials file: ".$serverCredentialsFileName);#Obviously don't ever print the _contents_ of the credentials file to the screen...
-
- $ftpServerName = $ftpCredentials->{"server"};
-
- #We handle the FTP being down, or we will keep retrying until it comes back up, but we will warn the MO if it is not up around the time the data is expected to be available
- my $ftp = Net::FTP->new($ftpServerName, Debug => 0) or handleDataNotAvailableServerDown($ftpServerName, $@);
-
- $ftp->login($ftpCredentials->{"user"}, $ftpCredentials->{"pass"}) or handleError("Cannot login ", $ftp->message);#NOTE: This error would be their fault, but we need to communicate properly to fix it
-
- $ftp->cwd("/Cambridge_wheatrust_Laura") or handleError("Cannot change working directory ", $ftp->message);#NOTE: This error would be their fault, but we need to communicate properly to fix it
-
- #Get the relevant zipped data file:
- $ftp->get($todayOutputFileName);
-
- $ftp->quit();
- }
-
- #Check it came down cleanly:
- unless(-e $todayOutputFileName) {
- #Not necessarily an error, probably just not available yet:
- if($debugOutput) {
- print("DEBUG: ".$debugTimeString." File not available: ".$todayOutputFileName);
- }
- #TODO: More prominent logging? Error if not available by a certain time? Explicit check for existence of file on the FTP server so we know difference between failed download and file not present yet?
- #TODO: TODAY Should email met office (+Cam team) if files are not present by $dataNotYetAvailableErrorTime (and make sure don't email multiple times...)
- handleDataNotAvailableByTime($ftpServerName);
-
- #Whether or not the data should have been here by now, we're done:
- terminateRun();
- }
-
- #Extract the zip to a folder:
- system("tar -xzf ".$todayOutputFileName);
-}
-
-#Check we got the folder we were expecting
-unless(-d $todayOutputDir) {
- handleError("ERROR: Didn't manage to unzip the output directory: ".$todayOutputDir." from ".$todayOutputFileName, \@maintainersAll);#Email the met office as well as the error is on their end -either totally empty zip or misnamed file
-}
-
-#Check if the zip actually had output inside it
-my $necessaryOutputsGlob = get_env_expected_inputs_glob();
-my @necessaryOutputs = glob($todayOutputDir.$necessaryOutputsGlob);
-
-if($debugOutput) {
- print("DEBUG: ".$debugTimeString." Building calls to gather part files\n");
-}
-my @gather_opts = ();
-my @rusttypes_to_gather = get_gather_parts_py_cmds();
-foreach my $rusttype (@rusttypes_to_gather) {
- push @gather_opts, '--inpath '.$todayOutputDir.'/'."\u$rusttype".'Rust_Ethiopia/ '.
- '--outpath '.$todayFolderPath.'/processed'.
- ' --rusttype '.lc($rusttype);
-}
-
-# process part files
-if($debugOutput) {
- print("DEBUG: ".$debugTimeString." Calling gather commands\n");
-}
-foreach my $gather_opt (@gather_opts) {
- system('/storage/app/EWS/General/EWS-Coordinator/run_EnvPostProcess.sh '.$gather_opt) and handleError("ERROR: Failed to gather parts");
-}
-if($debugOutput) {
- print("DEBUG: ".$debugTimeString." Gather commands complete\n");
-}
-
-my $plotPath = $todayFolderPath."/plotting/";
-
-if($debugNoPlot) {
- #handleError("No plotting taking place", \@maintainersCam, $severityStringWARNING);
- if($debugOutput) {
- print("DEBUG: No plotting taking place");
- }
-
- if($debugNoUpload) {
- # no more work to do, so clear up in-progress file
-
- #Successful run, no plotting expected at beginning of Feb 2020.
- # placeholder file state this
- my $noplotFilePath = $todayFolderPath."/".$jobIDString."_NO_PLOTTING.txt";
- open(my $noplotFH, ">", $noplotFilePath) or handleError("ERROR: Unable to open confirmation file for writing: ".$doneFilePath);
- print $noplotFH "Completed: ".getTimestampNow()."\n";
- close($noplotFH) or handleError("ERROR: Unable to write ".$noplotFilePath);
-
- #Flag overall success of operation:
- open(my $successFH, ">", $doneFilePath) or handleError("ERROR: Unable to open confirmation file for writing: ".$doneFilePath);
- print $successFH "Completed: ".getTimestampNow()."\n";
- close($successFH) or handleError("ERROR: Unable to write ".$doneFilePath);
-
- #Remove the in progress file and exit:
- terminateRun();
- }
-}
-
-unless($debugNoPlot) {
-
- # prepare for new plotting routine
- my @plottingCommands = get_env_plotting_cmds();
-
- foreach my $plotCommand (@plottingCommands) {
-
- # FALSE option so that hourly plots are not produced on first pass (to save time)
- #TODO: Handle 3hourly images later if needed
- my $plotOptions = $todayString." FALSE";
-
- # run the plot command
- system($plotCommand.' '.$plotOptions);
-
- # check there is some output
- unless(-d $plotPath) { handleError("ERROR: No plotting output produced")};
- }
-}
-
-#Plotting outputs in:
-
-my $plotOutputGlob = get_env_plotting_output_glob();
-my @plotPaths = glob($plotPath.$plotOutputGlob);
-
-#ENV outputs:
-my $localOutputFolderName = $todayString."_0000";
-
-#Raw data:
-#TODO: Confirm subset
-
-#Transfer the output to public server for others to be able to pull it down:
-unless($debugNoUpload) {
-
- # command to make remote directory
- my $willowDir1 = "/var/www/html/".$projectName."/".$localOutputFolderName;
- my $mkdir_cmd = "ssh -i ".$coordinatorPath."/ssh_key_willow willow.csx.cam.ac.uk \"mkdir -p ".$willowDir1."\"";
- print($mkdir_cmd);
- system($mkdir_cmd) and handleError("ERROR: Failed to create output directory on willow");
-
- # command to copy to remote directory
- foreach my $plotPathi (@plotPaths) {
- my $scp_cmd = "scp -i ".$coordinatorPath."/ssh_key_willow -r ".$plotPathi." willow.csx.cam.ac.uk:".$willowDir1."/";
- print($scp_cmd."\n");
- system($scp_cmd) and handleError("ERROR: Failed to copy plotting output to willow");
- }
-
- # Also transfer to sftp directory for others to automatically pull it down
- my $willowDir2 = "/storage/sftp/ewsguest/from-cam/".$projectName."/".$localOutputFolderName;
- $mkdir_cmd = "ssh -i ".$coordinatorPath."/ssh_key_willow willow.csx.cam.ac.uk \"mkdir -p ".$willowDir2."\"";
- print($mkdir_cmd);
- system($mkdir_cmd) and handleError("ERROR: Failed to create output directory on willow");
-
- # command to copy to remote directory
- foreach my $plotPathi (@plotPaths) {
- my $scp_cmd = "scp -i ".$coordinatorPath."/ssh_key_willow -r ".$plotPathi." willow.csx.cam.ac.uk:".$willowDir2."/";
- print($scp_cmd."\n");
- system($scp_cmd) and handleError("ERROR: Failed to copy plotting output to willow");
- }
-}
-
-#TODO: Bung todays workspace folder onto the RCS
-
-#Flag overall success of operation:
-open(my $successFH, ">", $doneFilePath) or handleError("ERROR: Unable to open confirmation file for writing: ".$doneFilePath);
-print $successFH "Completed: ".getTimestampNow()."\n";
-close($successFH) or handleError("ERROR: Unable to write ".$doneFilePath);
-
-#Successful run, remove the in-progress file and exit:
-terminateRun();
diff --git a/EnvProcessor.py b/EnvProcessor.py
deleted file mode 100644
index 60c2cb4..0000000
--- a/EnvProcessor.py
+++ /dev/null
@@ -1,604 +0,0 @@
-#EnvProcessor.py
-'''To be used for handling environmental suitability model, but can be
-generalised by changing the specification of process_pre_job(),
-process_in_job(), process_EWS_plotting(), perhaps by expanding the contents of
-the provided config file.
-
-Command-line options are used to complete the json config files.
-
-This runs all dates of a single forecast.
-
-
-Example usage::
- $ conda activate py3EWSepi
- $ python3 EnvProcessor.py --date 20200324 --islive --config config_Nepal_template_stripe.json
-or::
- $ ./run_EnvProcessor.sh --date 20200324 --islive --config config_Nepal_template_stripe.json
-'''
-import sys
-import os
-import glob
-import json
-import shutil
-
-import logging
-import logging.config
-
-import argparse
-from string import Template
-
-import datetime
-import subprocess
-from pathlib import Path
-
-import matplotlib as mpl
-import pandas
-import rasterio as rio
-
-from flagdir import jobStatus # gitlab project created by jws52
-
-print('Preparing for operational!')
-print("make sure to `conda activate py3EWSepi` environment!")
-print("make sure that flagdir package is available (on PYTHONPATH)")
-
-# TODO: specify which process_pre_job, process_in_job, and process_EWS_plotting to use in config file
-# TODO: Replace subprocess scp and ssh commands with paramiko.SSHClient() instance
-
-# initialise default values for configuration
-
-script_name = 'EnvProcessor'
-
-timeNow = datetime.datetime.today()
-dateToday = timeNow.date()
-todayString = timeNow.strftime('%Y%m%d')
-nowString = timeNow.strftime('%Y%m%d-%H%M-%S')
-
-# After today, 6th day from present is entirely included in forecast
-forecastLimit = timeNow + datetime.timedelta(days=6)
-forecastLimitString = forecastLimit.strftime('%Y%m%d')
-# configure default timespan of calculation
-# 6 whole days (consistent with what environmental suitability v1.1-1.3 can calculate)
-endTime = dateToday + datetime.timedelta(days=5,hours=21)
-# 6 hours
-#endTime = timeNow + datetime.timedelta(days=0,hours=3)
-# one whole day
-#endTime = timeNow + datetime.timedelta(days=0,hours=21)
-defaultEndString = endTime.strftime('%Y%m%d')
-
-log_path_project = '/storage/app/EWS/General/EWS-Coordinator/logs/log_env.txt'
-
-# log will be written here until a job directory exits, when it will be moved there
-log_path_default = f"/storage/app/EWS/General/EWS-Coordinator/logs/log_env_{nowString}.txt"
-
-maintainers = ['jws52@cam.ac.uk','tm689@cam.ac.uk','rs481@cam.ac.uk']
-
-with open('/storage/app/EWS/General/EWS-Coordinator/Cred_gmail.json','r') as f:
- gmailConfig = json.load(f)
-
-# load config from python dictionary (could be loaded from json file)
-# TODO: smtp handler can only use tls, but ssl is more secure. Look into defining/writing a suitable smtp handler
-logConfigDict = {
- 'version' : 1,
- 'disable_existing_loggers': False,
- 'formatters' : {
- 'simple' : {
- 'format' : '%(name)s : %(levelname)s - %(message)s'
- },
- 'detailed' : {
- 'format' : '''%(levelname)s in %(name)s:
- Encountered at %(asctime)s
-
- %(message)s
-
- Resolve this error and restart processing.''',
- 'datefmt' : '%Y-%m-%d %H:%M:%S'
- }
- },
- 'handlers' : {
-
- # logging for project
- 'handler_rot_file' : {
- 'class' : 'logging.handlers.TimedRotatingFileHandler',
- 'level' : 'INFO',
- 'formatter' : 'simple',
- 'filename' : log_path_project,
- # time of day on given day
- 'when' : 'W2', # rotate file every Wednesday
- 'atTime' : datetime.time(1,0,0), # at 1am local time
- 'backupCount' : 12,
- },
-
- # logging for job
- 'handler_file' : {
- 'class' : 'logging.FileHandler',
- 'level' : 'DEBUG',
- 'formatter' : 'simple',
- 'filename' : log_path_default,
- 'mode' : 'a', # 'a' for append
- },
-
- # to email errors to maintainers
- 'handler_email' : {
- 'class' : 'logging.handlers.SMTPHandler',
- 'level' : 'ERROR',
- 'mailhost' : (gmailConfig['host'], gmailConfig['port']), # host, port. 465 fsor SSL, 587 for tls
- 'credentials' : (gmailConfig['user'],gmailConfig['pass']),
- 'secure' : (), # passing a tuple enables tls
- 'timeout' : 1.0,
- 'fromaddr' : gmailConfig['user'],
- 'toaddrs' : maintainers,
- 'subject' : 'ERROR in ENV processing',
- 'formatter' : 'detailed',
- }
- },
- 'loggers' : {
- 'EnvProcessor' : { # this is activated with logging.getLogger('Process.')
- 'level' : 'DEBUG',
- 'handlers' : ['handler_rot_file','handler_file','handler_email'],
- 'propagate' : True,
- },
- '__main__' : { # this is activated with logging.getLogger(__name__) when name == '__main__'
- 'level' : 'DEBUG',
- 'handlers' : ['handler_rot_file','handler_file','handler_email'],
- 'propagate' : True,
- }
- }
-}
-
-logging.config.dictConfig(logConfigDict)
-
-print(__name__)
-# create a logger named according to how the file is called
-#logger = logging.getLogger(__name__)
-logger = logging.getLogger('EnvProcessor')
-
-def move_default_logfile_handler(dstPathName,srcPathName=log_path_default,FileHandlerName='handler_file',):
- '''For on-the-fly move of logging from default file to another. Copies the
- contents of the source log file to destination, switches file handler in
- logger, then removes source log file.'''
-
- # copy old log file to new filename
- srcPath = Path(srcPathName)
- dstPath = Path(dstPathName)
- assert srcPath.exists()
- assert dstPath.parent.is_dir()
-
- oldFileHandler = [h for h in logger.handlers if h.name==FileHandlerName][0]
- oldFormatter = oldFileHandler.formatter
-
- # define new file handler
- newfilehandler = logging.FileHandler(dstPath,mode=oldFileHandler.mode)
- newfilehandler.setLevel(oldFileHandler.level)
- newfilehandler.setFormatter(oldFormatter)
-
- logger.info(f"Moving logfile location from:\n{srcPathName}\nto:\n{dstPathName}")
- shutil.copyfile(srcPath,dstPath)
-
- # add handler for destination file
- logger.info('Adding new logging file handler to destination path')
-
- logger.addHandler(newfilehandler)
-
- # remove handler for source file
- logger.info('Stopping write to old file handler')
- logger.removeHandler(oldFileHandler)
- logger.info('Successfully stopped write to old file handler')
-
- # delete old log file
- logger.info('Deleting old log file, since all content available in new log file stream')
- os.rename(srcPathName,srcPathName+'removed')
-
- return
-
-def endScript(premature=True):
- if not premature:
- logger.info(f'{script_name} finished!')
- else:
- logger.info(f'{script_name} finished prematurely')
- logger.info(f'--------')
- sys.exit()
-
-def endJob(status,**kwargs):
- # get status file fresh, and ensure one exists
- status.status = status.get()
- assert status.has_status
- if status.is_inprogress():
- logger.warning(f'Workflow problem: {script_name} status is {status.status} but endJob called, forcing error status')
- status.reset('ERROR')
-
- endScript(**kwargs)
-
-def parse_and_check_args(todayString):
- # define the command line arguments
- my_parser = argparse.ArgumentParser(description='Command-line arguments for coordinator script of env suitability model')
-
- # Add the arguments
- # positional arguments do not start with - or -- and are always required
- # optional arguments start with - or -- and default is required = False
-
- my_parser.add_argument(
- '-c', '--config',
- metavar = 'path',
- type = str,
- nargs = '+', # require at least one path
- dest = 'config_paths',
- required = True,
- #default = ['config_Ethiopia_template_stripe.json'], # remove once live
- #default = ['config_Bangladesh_template_stripe.json'], # remove once live
- #default = ['config_Nepal_template_stripe.json'], # remove once live
- help = '''path to a config file(s). More than one can be provided,
- in which case each is worked on in turn (e.g. one each for stripe, stem, leaf).
- Do not place other options between these.''')
-
- my_parser.add_argument(
- '-l','--loglevel',
- action = 'store',
- default = 'debug',
- help = 'verbosity of log messaging (debug, info, warning, error, critical)\n default is debug',
- dest = 'log_level', # this names the attribute that will be parsed
- )
-
- my_parser.add_argument(
- '--islive',
- action = 'store_true',
- help = 'If live, email messages are sent to maintainers for warning and errors',
- dest = 'live',
- )
-
- my_parser.add_argument(
- '-s','--start-date','-i','--initial-date',
- metavar = 'YYYYMMDD',
- action = 'store',
- default = todayString,
- help = 'Initial day of calculation, starting at 00 UTC (Default is today)',
- dest = 'start_date',
- )
-
- my_parser.add_argument(
- '--noupload',
- action = 'store_true',
- help = 'whether results of script should be saved to willow public directory'
- )
-
- # get an object holding all of the args
- args = my_parser.parse_args()
-
- # Check the args
-
- logger.info(f"Command-line options are:\n{args}")
-
- if not args.live:
- # do not email maintainers
- logger.handlers = [h for h in logger.handlers if not isinstance(h,logging.handlers.SMTPHandler)]
-
- if not isinstance(args.config_paths,list):
- logger.error('Expecting a list of config paths')
- raise
-
- # check the startstring
- if args.start_date is not todayString:
- try:
- # check date string is formatted correctly
- provided_start_date = datetime.datetime.strptime(args.start_date,'%Y%m%d')
- today_date = datetime.datetime.strptime(todayString,'%Y%m%d')
-
- # early limit is quite arbitrary, but this is earliest year of available survey data for Ethiopia
- date_limit_early = datetime.datetime.strptime('20070101','%Y%m%d')
- assert date_limit_early < provided_start_date
- assert provided_start_date < today_date
-
- except (ValueError, AssertionError) as e:
- logger.exception("Provided start date string is formatted incorrectly or out of range, or end date not also defined")
- raise
-
- # modify log level if needed
- loglevels = {'debug':logging.DEBUG,
- 'info':logging.INFO,
- 'warning':logging.WARNING,
- 'error':logging.ERROR,
- 'critical':logging.CRITICAL,
- }
-
- logger.info(f"logging level being changed to {loglevels[args.log_level]} because of command-line option")
- logger.setLevel(loglevels[args.log_level])
-
- return args
-
-def open_and_check_config(configFile):
- logger.info("Loading config")
-
- try:
- with open(configFile) as config_file:
- config = json.load(config_file)
- except:
- logger.exception(f"Failure opening config file {configFile}")
- raise
-
- logger.debug(json.dumps(config,indent=2))
-
- #TODO: Checking config file for required formatting")
- try:
- pass
- except AssertionError as e:
- logger.exception("Configuration file is not formatted as required")
- raise
-
- return config
-
-def process_pre_job(input_args):
- '''This is set up for environmental suitability v2.0'''
-
- logger.info('started process_pre_job()')
-
- # Check if there is a file available on willow
- logger.debug('Checking for file(s) on remote server')
-
- for i,config_path in enumerate(input_args.config_paths):
-
- config = open_and_check_config(config_path)
-
- config['StartString'] = input_args.start_date
-
- file_path = Template(config['Environment']['ServerPathTemplate']).substitute(**config)
- file_name = Template(config['Environment']['InputFileTemplate']).substitute(**config)
- logger.info(f"Checking for existence of {file_path}/{file_name}.tar.gz")
-
- status = subprocess.run(["ssh","-i",config['ServerKey'],config['ServerName'],f"test -f {file_path}/{file_name}.tar.gz"])
-
- if status.returncode == 1:
- logger.info(f"Data not yet available for config {i}")
- endScript(premature=True)
-
- elif status.returncode == 0:
- logger.info(f"Data is available for config {i}, calculation shall proceed")
-
- return
-
-def process_in_job(jobPath,config):
- logger.info('started process_in_job()')
-
- logger.info('Copying file from remote server to job directory')
-
- file_path = Template(config['Environment']['ServerPathTemplate']).substitute(**config)
- file_name = Template(config['Environment']['InputFileTemplate']).substitute(**config)
-
- # TODO: perform ssh file transfer in python instead of subprocess
- subprocess.run(["scp","-i",config['ServerKey'],f"{config['ServerName']}:{file_path}/{file_name}.tar.gz", jobPath])
-
- logger.info('untarring the input file')
-
- # TODO: untar file in python (with tarfile module) instead of subprocess
- subprocess.run(["tar","-xzf",f"{jobPath}/{file_name}.tar.gz","-C",jobPath])
-
- # basic check that contents are as expected
- # 57 files of NAME .txt timesteps and one summary png file
- if len(os.listdir(f"{jobPath}/{file_name}")) != 58:
- logger.error(f"Insufficient contents of input tar file")
-
- # TODO: call envsuit v2.0
-
- # TODO: call EWS-Plotting
-
- pass
- return
-
-def process_EWS_plotting(config):
- logger.info('started process_EWS_plotting()')
-
- pass
- return []
-
-def process_EWS_plotting_epi(config):
-
- # initialise environment
- plot_path = '/storage/app/EWS/General/EWS-Plotting/'
- env_map = {
- 'PATH' : os.environ['PATH'],
- 'PYTHONPATH' : plot_path,
- 'R_LIBS' : '/home/ewsmanager/R-packages-EWS-plotting/plotting_rlibs/',
- }
- #sys.path.append(plot_path)
- #os.environ['R_LIBS'] = '/home/ewsmanager/R-packages-EWS-plotting/plotting_rlibs/'
-
- # TODO: prepare arguments
- epi_case_operational = config['EWS-Plotting']['EpiCase']
-
- if epi_case_operational == 'none':
- logger.info('Config specifies not to call to EWS-Plotting')
- return []
-
- python_script = config['EWS-Plotting']['PythonScript']
-
- run_config = config['EWS-Plotting']['RunConfig']
-
- epi_filename = [ce['infectionRasterFileName']+'.csv' for ce in config['Epi'] if ce['model']==epi_case_operational][0]
-
- deposition_dir = f"{config['WorkspacePath']}DEPOSITION_{startString}/WR_NAME_SouthAsia_{startString}/"
-
- ews_plot_dir = f"{jobPath}/plotting/"
-
- Path(ews_plot_dir).mkdir(parents=True, exist_ok=True)
-
- # prepare command
- plot_command = [
- '/storage/app/iris/irispython3.sh',
- python_script,
- '-epi',epi_filename,
- '-dt',config['DiseaseName'].lower(),
- '-ws',deposition_dir,
- '-o',ews_plot_dir,
- '-d',startString,
- '-sc','/storage/app/EWS/General/EWS-Plotting/python/data/json_config/SYS_CONFIG_PINE.json',
- '-rc',run_config]
-
- logger.info(f"Running EWS-Plotting command:\n'{' '.join(plot_command)}'")
-
- try:
- ews_plotting_process = subprocess.run(plot_command,
- env = env_map,
- check = True,
- stdout = subprocess.PIPE,
- stderr = subprocess.STDOUT)
-
- for line in ews_plotting_process.stdout.decode('utf-8').split(r'\n'):
- logger.info('EWS-Plotting: ' + line)
-
- except subprocess.CalledProcessError as e:
- logger.exception('EWS-Plotting did not produce any output')
- status.reset('ERROR')
- endJob(status,premature=True)
-
- # check the output
- EWSPlottingOutputDir = f"{ews_plot_dir}/epi/images/"
- EWSPlottingOutputGlobs = [f"{EWSPlottingOutputDir}{config['RegionName'].lower()}_{config['DiseaseName'].lower()}*.png"]
-
- # TODO: local substitute for EpiUtils.get_only_existing_globs(EWSPlottingOutputGlobs)
-
- # check there is some output from EWS-plotting
- if not EWSPlottingOutputGlobs:
- logger.error('EWS-Plotting did not produce any output')
- status.reset('ERROR')
- endJob(status,premature=True)
-
- # provide to list for transfer
- EWSPlottingOutputs = sorted(glob.glob(EWSPlottingOutputGlobs[0]))
-
- return EWSPlottingOutputs
-
-def run_Process():
-
- logger.info("==========")
- logger.info(f"Logging started at {datetime.datetime.now().strftime('%Y %b %d %H:%M:%S')}")
-
- # load configurations
-
- args = parse_and_check_args(todayString)
-
- # check initial state of each config file
-
- workspacePaths = []
- for configFile in args.config_paths:
- try:
- config_i = open_and_check_config(configFile)
- except:
- logger.exception(f"Failure in opening or checking config {configFile}")
- endScript(premature=True)
-
- # get workspace path from provided config files
- # TODO: Can this be done more neatly? Without needing a job config that lists separate args.config_paths?
- if config_i['WorkspacePathout'] not in workspacePaths:
- workspacePaths += [config_i['WorkspacePathout']]
-
- # If multiple configs, they must all have the same workspace path
- if len(workspacePaths) > 1:
- logger.error(f"config files point to multiple workspaces, can only handle one.")
- endScript(premature=True)
-
- workspacePath = workspacePaths[0]
-
- logger.info(f"Workspace path is {workspacePath}")
-
- # determine job directory
- jobPath = f'{workspacePath}ENVIRONMENT_2.0_{args.start_date}'
-
- logger.info(f"Job path will be {jobPath}")
-
- # note that file date represents preceding 3 hours, so day's data starts at file timestamp 0300 UTC
- startTime = datetime.datetime.strptime(args.start_date+'03','%Y%m%d%H')
-
- # run any checks before creating a job directory
- process_pre_job(args)
-
- # create job directory
- Path(jobPath).mkdir(parents=True, exist_ok=True)
-
- # lock job directory
- with jobStatus(jobPath) as status:
-
- # check for a status file in job directory
- if status.had_initial_status:
- logger.error(f"Job path already exists and has status {status.status}")
-
- endJob(status,premature = status.status != 'SUCCESS')
-
- logger.info(f"Current status of job directory is {status.status}")
-
- # now that we have a useable job directory, move the log file there
- logPathJob = f"{jobPath}/log.txt"
-
- move_default_logfile_handler(dstPathName=logPathJob)
-
- FilesToSend = []
-
- logger.info('Starting to work on each configuration')
-
- for configIndex, configtemplate in enumerate(args.config_paths):
-
- logger.info(f'Working on config {configIndex+1} of {len(args.config_paths)}')
-
- try:
- configjson = open_and_check_config(configtemplate)
- except:
- logger.exception(f"Failure in opening or checking config {configtemplate}")
- # TODO: This case should test flagdir.jobStatus.__exit__()
- raise # endJob('ERROR',premature=True)
-
- # provide specific case details to template config
-
- configjson['StartTime'] = startTime.strftime('%Y-%m-%d-%H%M')
- configjson['StartString'] = args.start_date
-
- # TODO: from configtemplate create configFileName to describe the specific job
- configFileName = f"{os.path.basename(configtemplate).replace('.json','')}_ENV_test"
-
- # TODO: fill in any missing details of config file
-
- # write the complete configuration file to job directory
- with open(f"{jobPath}/{configFileName}.json",'w') as write_file:
- json.dump(configjson,write_file,indent=4)
-
- process_in_job(jobPath,configjson)
-
- # Run EWS-plotting command
-
- EWSPlottingOutputs = process_EWS_plotting(configjson)
-
- logger.info('Finished with EWS-Plotting, appending images to list for transfer')
- FilesToSend += EWSPlottingOutputs
-
- logger.info(f'Finished with config {configIndex+1} of {len(args.config_paths)}')
-
- # send results to remote server
-
- # TODO: fix this when needed
- args.noupload = True
-
- if not args.noupload:
-
- # TODO: check path, may need to send Daily too
- OutputServerPath = f"{config['OutputServerPath']}/{config['StartString']}_0000/Weekly/"
-
- logger.info(f"Remote server is called {config['ServerName']}")
- logger.info(f"Path on remote server is {ServerPath}")
-
- logger.info(f"File(s) that will be put on remote server: {FilesToSend}")
-
- logger.debug("Making path directory on remote server if it doesn't already exist")
-
- subprocess.run(["ssh","-i",config['ServerKey'],config['ServerName'], f"mkdir -p {OutputServerPath}"])
-
- logger.debug('Sending file(s) to remote server')
-
- subprocess.run(["scp","-i",config['ServerKey'],*FilesToSend, f"{config['ServerName']}:{OutputServerPath}"])
-
- status.reset('SUCCESS')
-
- endScript(premature=False)
-
-try:
- run_Process()
-except SystemExit:
- pass
-except:
- logger.exception('Uncaught exception in run_Process:')
-
diff --git a/NAMEDataCheckerCron.sh b/NAMEDataCheckerCron.sh
deleted file mode 100755
index c88afbe..0000000
--- a/NAMEDataCheckerCron.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/bin/bash
-
-# Get paths
-coordpath="/storage/app/EWS/General/EWS-Coordinator/"
-
-configpath="$1"
-
-todaydate="$2"
-
-projectpath="$(perl -MPOSIX -e 'require $ARGV[0]; print get_project_path();' -- "$configpath")"
-
-# Start log and check paths exist
-logpath=${projectpath}Workspace/logs/
-logfile=${logpath}DEPDatalogs.txt
-
-if [[ ! -d ${logpath} ]]; then
- printf "Logfile directory does not exist"
- exit 1
-fi
-
-logstr1="=============\n"
-logstr1+="$( date ) \n"
-logstr1+="Looking for coordinator scripts in "${coordpath}"\n"
-logstr1+="Project configuration file is "${configpath}"\n"
-logstr1+="Project directory is "${projectpath}"\n"
-
-if [[ ! -d ${coordpath} ]]; then
- logstr1+="Coordinator directory does not exist"
- printf "${logstr1}" >> ${logfile}
- exit 1
-elif [[ ! -f ${configpath} ]]; then
- logstr1+="Configuration file does not exist"
- printf "${logstr1}" >> ${logfile}
- exit 1
-elif [[ ! -d ${projectpath} ]]; then
- logstr1+="Project directory does not exist"
- printf "${logstr1}" >> ${logfile}
- exit 1
-fi
-
-printf "${logstr1}" >> ${logfile}
-
-#Checks to see if todays NAME deposition data is available yet,
-#if so, pulls it down
-#and starts the portion of the EWS that runs on the Cambridge servers
-${coordpath}DataProcessor.pl ${configpath} ${todaydate} >> ${logfile} 2>&1
-
-# check if perl script ran cleanly, 0 means error
-if [ $? -ne 0 ]; then
- logstr2="ENVDataProcessor.pl failed to finish cleanly.\n"
- logstr2+="=============\n"
- printf "${logstr2}" >> ${logfile}
- exit 1
-fi
-
-logstr2="\n"
-logstr2+="SurveyDataProcessor.pl finished cleanly.\n"
-logstr2+="=============\n"
-printf "${logstr2}" >> ${logfile}
-exit 0
diff --git a/NAMEDataCheckerCrontab.txt b/NAMEDataCheckerCrontab.txt
deleted file mode 100644
index 5ab319b..0000000
--- a/NAMEDataCheckerCrontab.txt
+++ /dev/null
@@ -1 +0,0 @@
-*/5 * * * * /storage/app/EWS/General/EWS-Coordinator/NAMEDataCheckerCron.sh /storage/app/EWS/General/EWS-Coordinator/ProjectConfig_SouthAsia.pl
diff --git a/ProjectConfig_Ethiopia.pl b/ProjectConfig_Ethiopia.pl
deleted file mode 100644
index 4fd84f2..0000000
--- a/ProjectConfig_Ethiopia.pl
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/perl
-
-use warnings;
-use strict;
-
-sub get_project_name { return "Ethiopia" };
-sub get_project_path { return "/storage/app/EWS/Ethiopia/" };
-
-sub get_maintainers_cam { return ("jws52\@cam.ac.uk", "tm689\@cam.ac.uk", "rs481\@cam.ac.uk")};
-sub get_maintainers_all { return ("jws52\@cam.ac.uk", "tm689\@cam.ac.uk", "rs481\@cam.ac.uk", "william.thurston\@metoffice.gov.uk")};
-
-# Survey processing
-sub get_project_surveys_from_ODK { return 1 };
-sub get_project_ODK_credentials { return "/storage/app/EWS/Ethiopia/Workspace/Cred-ODK-EIAR.json" };
-sub get_project_historical_surveys { return "" };
-
-# Met office interaction
-sub get_download_from_willow{ return 1 };
-
-# NAME processing
-sub get_NAME_directory_prefix { return "NAME_wheatrust_" };
-sub get_NAME_file_prefixes { return ("depositionEthiopia_Stem_","depositionEthiopia_Stripe_") };
-sub get_expected_available_hour { return 8 };
-sub get_expected_available_minute { return 0 };
-
-# ENV processing
-sub get_ENV_directory_prefix { return "WR_EnvSuit_Ethiopia_" };
-sub get_env_expected_inputs_glob { return "{Stem,Stripe}Rust_Ethiopia/Region*/*0000/RIE_value.csv" }; # v1.3 output
-#sub get_env_expected_inputs_glob { return "?/RIE_value.csv" }; # v2.0 output
-sub get_gather_parts_py_cmds { return ('Stripe', 'Stem') };
-sub get_env_plotting_cmds { return (
- "/storage/app/EWS/General/EWS-Plotting-Temp/python/ethiopia/run_eth_env_plotting_pine.sh",) };
-sub get_env_plotting_output_glob {return "ethiopia/images/{Daily,Weekly}" };
-
-1;
\ No newline at end of file
diff --git a/ProjectConfig_SouthAsia.pl b/ProjectConfig_SouthAsia.pl
deleted file mode 100644
index fbb3868..0000000
--- a/ProjectConfig_SouthAsia.pl
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/perl
-
-use warnings;
-use strict;
-
-sub get_project_name { return "SouthAsia" };
-sub get_project_path { return "/storage/app/EWS/SouthAsia/" };
-
-sub get_maintainers_cam { return ("jws52\@cam.ac.uk", "tm689\@cam.ac.uk", "rs481\@cam.ac.uk")};
-sub get_maintainers_all { return ("jws52\@cam.ac.uk", "tm689\@cam.ac.uk", "rs481\@cam.ac.uk", "william.thurston\@metoffice.gov.uk")};
-
-# Survey processing
-sub get_project_surveys_from_ODK { return 1 };
-sub get_project_ODK_credentials { return "/storage/app/EWS/SouthAsia/Workspace/Cred-ODK-CIMMYT-Bangladesh.json" };
-# 2011-2018 composite of surveys in South Asia region,
-# after some manual modifications described in Rust_surveySouthEastAsia_19082019_curation_descriptions.txt
-# and after running survey_reformatter.py
-# prepared for 2019 only.
-# copy of /storage/app/SouthAsia-EWS/wheat-source-generation/data/RustSurveySouthEastAsia_19082019_curated_reformatted.csv
-# TODO: prepare this survey file for transition into year 2020.
-sub get_project_historical_surveys_filename { return "/storage/app/EWS/SouthAsia/Workspace/SURVEYDATA_MANUAL/HistoricalSurveyDataBlank.csv" };
-
-# Met office interaction
-sub get_download_from_willow{ return 1 };
-
-# NAME processing
-sub get_NAME_directory_prefix { return "WR_NAME_SouthAsia_" };
-sub get_NAME_file_prefixes { return ("deposition_srcs_allregions_") };
-sub get_expected_available_hour { return 8 };
-sub get_expected_available_minute { return 0 };
-sub get_dep_plotting_cmds { return (
- "/storage/app/EWS/General/EWS-Plotting/python/nepal/run_nepal_depo_plotting_pine.sh",
- "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/run_bang_depo_plotting_pine.sh") };
-
-# ENV processing
-sub get_ENV_directory_prefix { return "WR_EnvSuit_SouthAsia_" };
-sub get_env_expected_inputs_glob { return "{Stem,Stripe,Leaf}Rust_{Bangladesh,Nepal}/*0000/RIE_value.csv" };
-sub get_gather_parts_for { return () };
-sub get_env_plotting_cmds { return (
- "/storage/app/EWS/General/EWS-Plotting/python/nepal/run_nepal_env_plotting_pine.sh",
- "/storage/app/EWS/General/EWS-Plotting/python/bangladesh/run_bang_env_plotting_pine.sh") };
-sub get_env_plotting_output_glob {return "{bangladesh,nepal}/images/{Daily,Weekly}" };
-
-# visualisation
-
-1;
diff --git a/SurveyDataCheckerCrontab.txt b/SurveyDataCheckerCrontab.txt
deleted file mode 100644
index 5df7987..0000000
--- a/SurveyDataCheckerCrontab.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-55 22 * * * /storage/app/EWS/General/EWS-Coordinator/SurveyDataProcessor.pl --config /storage/app/EWS/General/EWS-Coordinator/ProjectConfig_SouthAsia.pl --live --verbose
-
diff --git a/SurveyDataProcessor.pl b/SurveyDataProcessor.pl
deleted file mode 100755
index be1878b..0000000
--- a/SurveyDataProcessor.pl
+++ /dev/null
@@ -1,582 +0,0 @@
-#!/usr/bin/perl
-
-use warnings;
-use strict;
-
-use POSIX qw(strftime);
-use DateTime;
-use JSON::MaybeXS;
-use File::Slurp;
-use File::Copy::Recursive qw(fcopy rcopy dircopy fmove rmove dirmove);
-use File::Basename;
-use Getopt::Long;
-
-# providing default list to email
-my @maintainers = ("jws52\@cam.ac.uk", "tm689\@cam.ac.uk", "rs481\@cam.ac.uk");
-
-# until command-line option is known, assume to send email notifications
-my $isliveearly = 1; # true. whether to email maintainers of errors and warnings before option is checked.
-
-#Add fixed timestamp generated at the start of call to outputs so possible to distinguish between different runs of the script
-sub getTimestampNow {
- return strftime("%Y%m%d%H%M", localtime());
-}
-my $debugTimeString = getTimestampNow();
-
-my $todayDate = DateTime->today(time_zone => "UTC");
-my $todayString = $todayDate->strftime("%Y%m%d");
-
-my $coordinatorPath="/storage/app/EWS/General/EWS-Coordinator/";
-unless(-d $coordinatorPath) {
- die "Coordinator path does not exist";
-}
-
-# if there are any errors before successfully loading config file, write in this log file
-my $logFileGeneral = $coordinatorPath."logs/SurveyLogs_before_configured.txt";
-open(my $logFH, '>>', $logFileGeneral) or die "Could not open '$logFileGeneral' $!";
-*STDOUT = $logFH;
-*STDERR = $logFH;
-my $redirectString = " >>".$logFileGeneral." 2>&1";
-
-# for verbose/debug messages before the project-specific log file is opened, build list
-my $earlyLog = "";
-sub printearly {
- my $printString = shift;
- $earlyLog .= $printString;
-}
-
-# initialise log
-printearly("=====\n");
-printearly($debugTimeString."\n");
-printearly("Surveydataprocessor.pl starting.\n");
-printearly("Note that before options are parsed, errors will always be emailed to maintainers.\n");
-
-# for errors before the project options and config are loaded
-# Send email notification if required, write to log file
-sub handleEarlyError {
- my $errorString = shift;
-
- my $failTimeStamp = getTimestampNow();
-
- #Send email notifications:
- if ($isliveearly) {
- my $emailCmd = "/storage/app/servermail/servermail/servermail.sh";
- my $emailSubject = "ERROR in the EWS SurveyDataProcessor.pl ".$failTimeStamp;
- # Write each line of email separately
- my @emailTexts = ("Error encountered at ".$failTimeStamp." during running of SurveyDataProcessor.pl",
- $errorString,
- "Any automatic running has not been suspended, so will probably keep failing.",
- "Resolve this error and re-enable automatic processing.");
- my $emailText = "";
- # Gather lines into one string
- foreach my $emailLine (@emailTexts) {
- $emailText = $emailText.$emailLine."\n\n";
- }
- # run the email command
- foreach my $maintainer (@maintainers) {
- system($emailCmd." -r ".$maintainer." -s \"".$emailSubject."\" -m \"".$emailText."\"".$redirectString);
- }
- }
-
- # print verbose/debug messages so far.
- # Once project-specific logfile picked up, messages will go there.
- print $earlyLog;
- die "======\n".$failTimeStamp."\n".$errorString;
-}
-
-# initialise optional arguments with defaults
-my $configFilePath; # this is made mandatory later
-my $dateString = $todayString; # run for today
-my $islive = 0; # false. whether to email maintainers of errors and warnings.
-my $verbose = 0; # false. whether to print additional messages
-my $debugNoUpload = 0; # false. whether to send results to willow
-my $debugNoDownload = 0; # false. whether to download from server, or use previous day's download
-
-# configFilePath is an optional argument followed by a string-type value
-# date is an optional argument followed by a string-type value
-# live is a boolean switch
-# verbose or v is a boolean switch
-# noupload is a boolean switch
-# nodownload is a boolean switch
-
-my $getoptErrorString = "ERROR: comand line options formatted incorrectly, example usage:\n";
-$getoptErrorString .= "SurveyDataProcessor.pl --config configfile.pl --date 20070615 --live --noupload --nodownload";
-
-GetOptions ('config=s' => \$configFilePath,
- 'date=s' => \$dateString,
- 'live' => \$islive,
- 'verbose|v' => \$verbose,
- 'noupload' => \$debugNoUpload,
- 'nodownload' => \$debugNoDownload
- ) or handleEarlyError($getoptErrorString);
-
-$isliveearly = $islive;
-
-# make config option mandatory
-unless (defined $configFilePath && -e $configFilePath ) {
- handleEarlyError("config option must be defined and point to existing perl file.");
-}
-if ($verbose){
- printearly("DEBUG: ".$debugTimeString." Verbose setting, additional messages will be printed.\n");
-
- if ($debugNoUpload) {
- printearly("DEBUG: ".$debugTimeString." Results will not be uploaded to willow.\n");
- }
-
- if ($debugNoDownload) {
- printearly("DEBUG: ".$debugTimeString." No download from server, will try to use latest available download.\n");
- }
-
- if ($islive) {
- printearly("DEBUG: ".$debugTimeString." This is live, so any warnings or errors will be emailed to maintainers.\n");
- }
-}
-
-# check date
-
-# parse the input date as DateTime
-my ($y,$m,$d) = $dateString =~ /^([0-9]{4})([0-9]{2})([0-9]{2})\z/ or handleEarlyError("Bad date string. Expecting YYYYMMDD");
-my $dateDate = DateTime->new(
- year => $y,
- month => $m,
- day => $d,
- time_zone => 'local') or handleEarlyError("Mis-handled date string");
-
-if ( $dateString != $todayString){
- if ($verbose){
- printearly("DEBUG: ".$debugTimeString."datestring is not equal to todaystring\n");
- }
-
- # set an earliest date allowed for calculations (quite arbitrary for now)
- # 2007 is earliest year of surveys for Ethiopia on rusttracker.cimmyt.org
- my $dateLimit = DateTime->new(year=>2007,month=>1,day=>1);
-
- my $cmp1 = DateTime->compare($dateLimit,$dateDate);
- my $cmp2 = DateTime->compare($dateDate,$todayDate);
-
- unless($cmp1 == -1){handleEarlyError("Supplied date is earlier than 2005.")}
-
- if($cmp2 == 1){handleEarlyError("Supplied date is in the future!")}
-
-}
-if ($verbose){
- printearly("DEBUG: ".$debugTimeString." Running for ".$dateString."\n");
-}
-
-# load configuration
-require $configFilePath or handleEarlyError("Could not load config file");
-
-# file should contain the following perl subroutines
-my @requiredconfigs = ("get_project_name",
- "get_project_path",
- "get_project_surveys_from_ODK",
- "get_project_ODK_credentials",
- "get_maintainers_cam",
- "get_project_historical_surveys_filename",
- );
-
-# check contents of config are as required
-foreach my $configname (@requiredconfigs) {
- unless (defined &{$configname}) {
- handleEarlyError("Config file ".$configFilePath." is missing configuration called ".$configname.". Please provide in file and re-run.");
- }
-}
-
-# TODO: Check contents of config subroutines are as required
-# TODO: Consider easier format for config file.
-
-#Get variables saved in project configuration
-my $projectName = get_project_name() or handleEarlyError("Cannot get project name from config file");
-my $projectPath = get_project_path() or handleEarlyError("Cannot get project path from config file");
-my $SurveysFromServer = get_project_surveys_from_ODK() or handleEarlyError("Cannot survey source from config file");
-my $ODK_Credentials = get_project_ODK_credentials() or handleEarlyError("Cannot get survey server credentials from config file");
-#my $SurveysFromServer = 0; # 1 means we download from ODK, 0 means we use a composite of historical surveys
-# TODO: replace with reference to alternative function instead, where each function can get surveys differently (e.g. ODK, other server pull, historical composite)
-
-# workspace is in project directory
-my $workspacePath = $projectPath."/Workspace";
-
-my $RPath = "/usr/local/R/bin/Rscript";
-my $RPath2 = "";
-
-# Start project-specific log and check paths exist
-my $logFileProject=$projectPath."/Workspace/logs/SurveyDatalogs.txt";
-open(my $projectlogFH, '>>', $logFileProject) or handleEarlyError("Could not open '$logFileProject' $!");
-*STDOUT = $projectlogFH;
-*STDERR = $projectlogFH;
-$redirectString = " >>".$logFileProject." 2>&1"; # for system calls
-close $logFH;
-
-# provide the earlier verbose/debug messages to the project-specific log
-print $earlyLog;
-
-@maintainers = get_maintainers_cam() or print("Cannot get list of maintainers from config file so using default list");
-
-if($verbose) {
- print("DEBUG: ".$debugTimeString." Maintainers are: ");
- foreach my $maintainer (@maintainers) {
- print($maintainer." ");
- }
- print("\n");
-}
-
-if($verbose) {
- print("DEBUG: ".$debugTimeString." Calculating surveys for ".$dateString."\n");
-}
-
-my $jobIDString = "SURVEYDATA";
-
-my $todayFolderPath = $workspacePath."/".$jobIDString."_".$dateString;
-
-my $inProgressFilePath = $todayFolderPath."/".$jobIDString."_INPROGRESS.txt";
-my $doneFilePath = $todayFolderPath."/".$jobIDString."_SUCCESS.txt";
-my $failFilePath = $todayFolderPath."/".$jobIDString."_ERROR.txt";
-my $warningFilePath = $todayFolderPath."/".$jobIDString."_WARNING.txt";
-
-# Send email notification if required, write to log file, and handle any lock files
-sub handleError {
-
- my $errorString = shift;
-
- my $failTimeStamp = getTimestampNow();
-
- #Send email notifications:
- if ($islive) {
- my $emailCmd = "/storage/app/servermail/servermail/servermail.sh";
- my $emailSubject = "ERROR in the ".$projectName." EWS ".$jobIDString." ".$failTimeStamp;
- # Write each line of email separately
- my @emailTexts = ("Error encountered at ".$failTimeStamp." during running of ".$jobIDString." initiated at ".$debugTimeString,
- $errorString,
- "Automatic running of ".$jobIDString." has been suspended.",
- "Resolve this error and remove the error file ".$failFilePath." to reenable automatic processing of ".$jobIDString."\"");
- my $emailText = "";
- # Gather lines into one string
- foreach my $emailLine (@emailTexts) {
- $emailText = $emailText.$emailLine."\n\n";
- }
- # run the email command
- foreach my $maintainer (@maintainers) {
- system($emailCmd." -r ".$maintainer." -s \"".$emailSubject."\" -m \"".$emailText."\"".$redirectString);
- }
- }
-
- #Log error with error file:
- open(my $failFH, ">", $failFilePath) or die "ERROR: ".$debugTimeString." Unable to open error file for writing: ".$failFilePath." for error ".$errorString;
- print $failFH "Failed: ".$failTimeStamp."\n";
- print $failFH $errorString."\n";
- close($failFH) or die "ERROR: ".$debugTimeString." Unable to write ".$failFilePath." for error ".$errorString;
-
- #Remove in-progress flag, how likely to succeed if there has been an error somewhere...
- if(-e $inProgressFilePath) {
- unlink($inProgressFilePath)
- }
-
- die $errorString;
-}
-
-sub handleWarning {
- my $warningString = shift;
-
- my $warningTimeStamp = getTimestampNow();
-
- $warningString = "Warning encountered at ".$warningTimeStamp." during running of ".$jobIDString." initiated at ".$debugTimeString.":\n".$warningString."\n";
-
- my $warningSubject = "WARNING in the ".$projectName." EWS ".$jobIDString." ".$warningTimeStamp;
-
- #Send email notifications:
- if ($islive) {
- foreach my $maintainer (@maintainers) {
- system("/storage/app/servermail/servermail/servermail.sh -r ".$maintainer." -s \"".$warningSubject."\" -m \"".$warningString."\"".$redirectString);
- }
- }
-
- #Log warning with warning file:
- open(my $warningFH, ">>", $warningFilePath) or die "ERROR: ".$debugTimeString." Unable to open warning file for writing: ".$warningFilePath." for warning: ".$warningString;
- print $warningFH $warningString;
- close($warningFH) or die "ERROR: ".$debugTimeString." Unable to write to ".$warningFilePath." for warning: ".$warningString;
-
-}
-
-#Transfer has errored out already, needs intervention to fix it:
-if(-e $failFilePath) {
- if($verbose) {
- print("DEBUG: ".$debugTimeString." ERROR file (".$failFilePath.") already exists, stopping early pending intervention to resolve issue.\n");
- }
- exit();
-}
-
-#Make sure our destination exists if not already present:
-unless(-d $todayFolderPath) {
- mkdir($todayFolderPath) or handleError("ERROR: Unable to create output folder path: ".$todayFolderPath);
-}
-
-#Use the existence of the done file folder as the signal that we have already done the work for today:
-if(-e $doneFilePath) {
- #We have already done the extraction for today:
- if($verbose) {
- print("DEBUG: ".$debugTimeString." Already done for today\n");
- }
- exit();
-}
-
-if(-e $inProgressFilePath) {
- #We already have a job under way:
- if($verbose) {
- print("DEBUG: ".$debugTimeString." Already a job in progress\n");
- }
- exit();
-} else {
- #Create the in progress file
- open(my $progressFH, ">", $inProgressFilePath) or handleError("ERROR: Unable to open ".$jobIDString." in-progress file for writing: ".$inProgressFilePath);
- print $progressFH "Started: ".getTimestampNow()."\n";
- close($progressFH) or handleError("ERROR: Unable to write ".$inProgressFilePath);
-
- # initiate log
- print("--------------\n");
- print($debugTimeString."\n");
- print("SurveyDataProcessor.pl in progress!\n")
-}
-
-#Enter destination for the files:
-chdir($todayFolderPath) or handleError("ERROR: Unable to change working directory to output folder path :".$todayFolderPath);
-
-my $serverFailure = 0;
-
-my $csvOutputDir = $todayFolderPath."/ExportCSV";
-my $csvFileName = "SurveyData.csv";
-
-# initialising some variables so that their scope is global
-my $downloadMethodString = "";
-my $ODK_FormName = "";
-
-if ($SurveysFromServer) {
- # get data from ODK server
-
- #credentials stored separately so they aren't in plain text and checked in to the repo:
- my $serverCredentialsFileName = $ODK_Credentials;
- my $serverCredentialsString = read_file($serverCredentialsFileName) or handleError("ERROR: Unable to read server credentials from credentials file: ".$serverCredentialsFileName);
- my $serverCredentials = decode_json($serverCredentialsString) or handleError("ERROR: Unable to parse credentials from JSON in credentials file: ".$serverCredentialsFileName);#Obviously don't ever print the _contents_ of the credentials file to the screen...
-
- my $serverJar = $coordinatorPath."/ODK-Briefcase-v1.11.2.jar";
- my $serverName = $serverCredentials->{"server"};
- my $serverFormID = $serverCredentials->{"form_id"};
- $ODK_FormName = $serverCredentials->{"form_name"};
-
- my $serverOutputDir = $todayFolderPath."/ExportRawDB";
-
- $downloadMethodString = "ODK database ".$serverName;
-
-
- unless($debugNoDownload) {
-
- #It appears that outages are normally extended, so in this case we will just have to take the most recent successful output
- #as it is unlikely to resolve itself in time for the forecast runs to start
-
- my $getDataCmd = "java -jar ".$serverJar." --pull_aggregate --form_id ".$serverFormID." --storage_directory ".$serverOutputDir." --aggregate_url ".$serverName." --odk_username ".$serverCredentials->{"user"}." --odk_password ".$serverCredentials->{"pass"};
- system($getDataCmd.$redirectString);
-
- #Check it came down cleanly ($serverOutputDir is created whether cleanly or not, so test more explicitly):
- if(-d $serverOutputDir."/ODK Briefcase Storage/forms/".$ODK_FormName) {
-
- # export the downloaded ODK database to csv
- my $dataToCSVCmd = "java -jar ".$serverJar." --export --form_id ".$serverFormID." --storage_directory ".$serverOutputDir." --export_directory ".$csvOutputDir." --export_filename ".$csvFileName;
- system($dataToCSVCmd.$redirectString);
-
- } else {
- #An error, the survey data should always be available
- #May have to revise this to a warning depending on how reliable the server is
-
- if($verbose) {
- print("DEBUG: ".$debugTimeString." survey data not downloaded: ".$serverOutputDir. "not present.\n");
- }
-
- $serverFailure = 1;
- #handleError("ERROR: Unable to obtain the survey data from ".$serverName);
- }
- }
-} else {
- $downloadMethodString = "fixed historical csv";
- # get copy of csv
-}
-
-#TODO: We probably can't check for the output actually existing cleanly, as folder names aren't the same as form names, so just rely on the next export step failing
-
-#Make sure our destination exists if not already present:
-unless(-d $csvOutputDir) {
- mkdir($csvOutputDir) or handleError("ERROR: Unable to create output folder path: ".$csvOutputDir);
-}
-
-# get survey data as csv
-unless($serverFailure || $debugNoDownload) {
-
- unless($SurveysFromServer) {
- # get a copy of the fixed historical csv file
- my $originalcopyFileName = get_project_historical_surveys_filename() or handleError("Cannot get filename of historical surveys from config file");
-
- # check if the fixed historical file exists
- unless (-e $originalcopyFileName) {
- handleError("ERROR: The file of fixed historical surveys specified in project config does not exist")
- }
-
- my $todaysCopy = $csvOutputDir."/".$csvFileName;
-
- fcopy( $originalcopyFileName, $todaysCopy ) or handleError("ERROR: Failed to copy fixed survey file: ".$originalcopyFileName);
- }
-
- unless(-e $csvOutputDir."/".$csvFileName) {
- if($verbose) {
- print("DEBUG: ".$debugTimeString." survey data not converted to csv successfully: ".$csvOutputDir);
- }
-
- $serverFailure = 1;
- #handleError("ERROR: Unable to obtain the survey data from ".$serverName);
- }
-
-}
-
-if ($serverFailure || $debugNoDownload) {
- #Server was not able to provide us with the latest data, so we need to look back in time until we find one that succeeded
-
- my $downloadFailString = "ERR_NO_ODK_DATABASE";
- my $lastSuccessfulDownloadtocsvFile = $downloadFailString;
-
- # TODO: Enable lookback for historical dates
- my $epoc = time();#a bit fragile if we've gone into a new day since the start of the script (highly unlikely as script takes about a second to execute), but we're looking back 7 days, and looking at "today" again and only going back 6 is still fine
-
- my $maxLookbackDays = 7;
-
- my $staleDownloadDataWarningString = "";
-
- for(my $nLookbackDays = 1; $nLookbackDays <= $maxLookbackDays; $nLookbackDays++) {
-
- my $oldTime = $epoc - $nLookbackDays * 24 * 60 * 60;#Apparently how we're meant do do this in perl, subtracting 24 hours worth of seconds from the time...
-
- my $olddayString = strftime("%Y%m%d", gmtime($oldTime));
-
- my $testDownloadPath = $workspacePath."/".$jobIDString."_".$olddayString."/ExportRawDB/ODK Briefcase Storage/forms/".$ODK_FormName;
-
- my $testDownloadtocsvOutputFile = $workspacePath."/".$jobIDString."_".$olddayString."/ExportCSV/SurveyData.csv";
-
- my $testSurveySuccessFile = $workspacePath."/".$jobIDString."_".$olddayString."/".$jobIDString."_SUCCESS.txt";
-
- if(-e $testDownloadtocsvOutputFile && -d $testDownloadPath && -e $testSurveySuccessFile) {
-
- $lastSuccessfulDownloadtocsvFile = $testDownloadtocsvOutputFile;
-
- $staleDownloadDataWarningString = "WARNING: Unable to obtain ".$downloadMethodString." for today, so is using the most recent successful download from ".$olddayString.", ".$nLookbackDays." days ago.";
-
- last;#perl for "break"
- }
- }
-
- if(!($lastSuccessfulDownloadtocsvFile eq $downloadFailString)) {
-
- print "Transferring old output\n".$staleDownloadDataWarningString." from folder ".$lastSuccessfulDownloadtocsvFile."\n";
-
- fcopy($lastSuccessfulDownloadtocsvFile, $csvOutputDir."/".$csvFileName);
-
- my $warningTimeStamp = getTimestampNow();
-
- handleWarning($staleDownloadDataWarningString, $warningTimeStamp);
- } else {
- handleError("ERROR: Unable to find successfully downloaded ".$downloadMethodString." output within ".$maxLookbackDays." days prior to ".$dateString);
- }
- $serverFailure = 0;
-}
-
-my $processedSurveyDataFileName = $csvOutputDir."/Processed_".$csvFileName;
-
-my $surveyDataErrorsToRemoveFileName = $workspacePath."/SURVEYDATA_MANUAL/SurveyDataErrorsToRemove.csv";
-my $surveyDataAdditionalDataFileName = $workspacePath."/SURVEYDATA_MANUAL/LIVE_SURVEYDATA_TOUSE.csv";
-
-#Check we got the folder we were expecting, this should happen whether ODK download is successful or not
-if (-d $csvOutputDir) {
- #Do any necessary modifications to the survey data:
- my $surveyOriginalFileName = $csvOutputDir."/".$csvFileName;
- if(0){
- #DEBUG:
- #fcopy($surveyOriginalFileName, $processedSurveyDataFileName);
- }
-
- my $preprocessCmd = $RPath." ".$coordinatorPath."/SurveyDataPreprocessor.R ".$surveyOriginalFileName." ".$surveyDataErrorsToRemoveFileName." ".$surveyDataAdditionalDataFileName." ".$processedSurveyDataFileName;
- #print($preprocessCmd);
- system($preprocessCmd.$redirectString);
-} else {
- #TODO: If this fails, we should either retry after a few minutes, or just put up yesterdays data
- if($verbose) {
- print("DEBUG: ".$debugTimeString." csv Files not extracted: ".$csvOutputDir);
- }
- handleError("ERROR: Didn't manage to process survey csv file in output directory: ".$csvOutputDir);
-}
-
-#Clustering analysis:
-# TODO: generaise the wheat-source-generation scripts. Currently the shapefiles contained are region-specific.
-my $clusterAnalysisPath = $projectPath."wheat-source-generation";
-
-chdir($clusterAnalysisPath) or handleError("ERROR: Unable to change working directory to clustering folder path :".$clusterAnalysisPath);
-
-#Clear old outputs
-my $oldClusteringOutputGlob = $clusterAnalysisPath."/output/sources_*";
-my @oldClusteringOutputs = glob($oldClusteringOutputGlob);
-
-foreach my $oldOutputFile (@oldClusteringOutputs) {
- unlink($oldOutputFile);
-}
-
-#Just in case the server was *very* slow to send the data
-my $todayAndrewString = strftime("%Y-%m-%d", localtime());
-
-my $clusteringCmd = "export R_LIBS=/home/ewsmanager/R-packages-EWS-clustering/x86_64-pc-linux-gnu-library/3.5; ".$RPath." code/R/clustering.R ".$processedSurveyDataFileName." ".$dateString." -2 7 ".$redirectString."; unset R_LIBS";
-if($verbose) {
- print "DEBUG:\n";
- print $clusteringCmd."\n";
-}
-system($clusteringCmd);
-
-#Find the output file:
-my $clusteringOutputPathGlob = $clusterAnalysisPath."/output/sources_".$todayAndrewString."_*.csv";
-my @clusteringOutputs = glob($clusteringOutputPathGlob);
-
-#Check output for successful completion:
-if(scalar(@clusteringOutputs) == 0) {
- handleError("ERROR: No output produced from clustering in ".$clusteringOutputPathGlob);
-}
-
-#We have cleared old outpus, so we should have exactly one now
-if(scalar(@clusteringOutputs) > 1) {
- handleError("ERROR: Multiple outputs found from clustering in ".$clusteringOutputPathGlob);
-}
-
-#We are happy that there is exactly one output file:
-my $clusteringOutputFile = $clusteringOutputs[0];
-
-#Uploading data:
-my $localOutputFolderName = $todayFolderPath."/".$jobIDString."_".$dateString."_0000";
-
-mkdir($localOutputFolderName) or handleError("ERROR: Unable to create the output upload staging folder ".$localOutputFolderName);
-
-my $sourcesUploadFileName = $localOutputFolderName."/sources_".$dateString.".csv";
-fcopy($clusteringOutputFile, $sourcesUploadFileName) or handleError("ERROR: Unable to copy output file ".$clusteringOutputFile." to the output upload staging folder ".$sourcesUploadFileName);
-
-#Upload the survey data:
-unless($debugNoUpload) {
- #Transfer the output to public server for others to be able to pull it down:
- system("scp -i ".$coordinatorPath."/ssh_key_willow"." -r ".$localOutputFolderName." willow.csx.cam.ac.uk:/var/www/html/".$projectName."/".$redirectString);
-
- # Also transfer to sftp directory for others to automatically pull it down
- system("scp -i ".$coordinatorPath."/ssh_key_willow"." -r ".$localOutputFolderName." willow.csx.cam.ac.uk:/storage/sftp/metofficeupload/upload/".$projectName."/toMO/".$redirectString);
-}
-
-#Successful exit, remove the in progress file:
-unlink($inProgressFilePath) or handleError("ERROR: Unable to remove in-progress lock file: ".$inProgressFilePath);
-
-#Flag overall success of operation:
-open(my $successFH, ">", $doneFilePath) or handleError("ERROR: Unable to open confirmation file for writing: ".$doneFilePath);
-print $successFH "Completed: ".getTimestampNow()."\n";
-close($successFH) or handleError("ERROR: Unable to write ".$doneFilePath);
-
-if($verbose) {
- print("Successfully completed: ".getTimestampNow()."\n");
-}
-close $projectlogFH;
-exit();
diff --git a/TEST_NAME_Processing.pl b/TEST_NAME_Processing.pl
deleted file mode 100755
index f8334ce..0000000
--- a/TEST_NAME_Processing.pl
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/perl
-
-use warnings;
-use strict;
-
-use JSON::MaybeXS;
-use File::Slurp;
-use DateTime;
-use DateTime::Format::HTTP;
-
-my $startDateString = "2018-12-15";
-my $nDays = 7;#This doesn't seem to do anything, it seems to just run for 7 days anyway...
-
-my $class = 'DateTime::Format::HTTP';
-my $startDt = $class->parse_datetime($startDateString);
-
-my $endDt = $startDt->add(days =>$nDays);
-
-my $endDateString = $endDt->strftime("%Y-%m-%d");
-
-print($endDateString."\n");
-
-my $plottingPath = "/storage/app/Ethiopia-EWS/Plotting/";
-
-my $cfgSubPath = "config/rs481/";
-
-my $cfgFileName = "NAME_plotting.json";
-
-my $cfgFileFullPath = $plottingPath."/".$cfgSubPath."/".$cfgFileName;
-
-my $cfgTemplateFileFullPath = $plottingPath."/".$cfgSubPath."/TEMPLATE_".$cfgFileName;
-
-my $cfgString = read_file($cfgTemplateFileFullPath) or die $!;
-my $cfgJSON = decode_json($cfgString) or die $!;
-
-#Autogenerate a unique output folder name:
-my $outDirInd = 0;
-my $outDirName = "";
-do {
- $outDirName = "output/TEST_NAMEOUTPUT_".$startDateString."_".$outDirInd;
- $outDirInd++;
-} while (-d $plottingPath.$outDirName);
-
-$cfgJSON->{"OutputDirectory"} = $outDirName;
-$cfgJSON->{"PastDataExtractionStartDate"} = $startDateString;
-$cfgJSON->{"nDaysExtraction"} = $nDays;#seemingly pointless
-$cfgJSON->{"PastDataExtractionEndDate"} = $endDateString;#seemingly pointless
-my $NAMEDateString = $startDateString =~ s/-//gr;
-$cfgJSON->{"NOTDAN_InputDirectory"} = "/storage/app/Ethiopia-EWS/Ethiopia-EWS-Workspace/DEPOSITION_TESTING/NAME_species_wheatrust_".$NAMEDateString;
-
-my $json = JSON::MaybeXS->new(utf8 => 1, pretty => 1);
-
-my $cfgStringOut = $json->encode($cfgJSON) or die $!;
-
-write_file($cfgFileFullPath, $cfgStringOut)or die $!;
-
-chdir($plottingPath) or die $!;
-
-my $cmd = "/usr/local/R/bin/Rscript Submit_mass_Gilligan.R ".$cfgSubPath."/".$cfgFileName." --fastanimation --NAME";
-print $cmd."\n";
-print `$cmd`;
-
-print "\n\n";
-print "If there were no errors, your output should now be present in:\n";
-print $plottingPath."animation_images/".$cfgJSON->{"OutputDirectory"}."/".$NAMEDateString."0000\n";
diff --git a/testClearTodayData.pl b/testClearTodayData.pl
deleted file mode 100755
index 05fba86..0000000
--- a/testClearTodayData.pl
+++ /dev/null
@@ -1,166 +0,0 @@
-#!/usr/bin/perl
-
-use warnings;
-use strict;
-
-use POSIX qw(strftime);
-use File::Path qw(make_path remove_tree);
-use Getopt::Long;
-use File::Copy::Recursive qw(fcopy rcopy dircopy fmove rmove dirmove);
-
-#This process would be so trivial as to not need a script, if only the plotting had actually configurable paths...
-
-print STDERR "testClearTodayData.pl SHOULD NOT BE CALLED IN PRODUCTION!\n";
-
-my $workspacePath = "/storage/app/Ethiopia-EWS/Ethiopia-EWS-Workspace";
-
-my $todayString = strftime("%Y%m%d", localtime());
-
-my $clearEnv = 0;
-my $clearNAME = 0;
-my $clearSurvey = 0;
-
-my $foo = GetOptions(
- "environment" => \$clearEnv,
- "NAME" => \$clearNAME,
- "survey" => \$clearSurvey,
- "date=s" => $todayString
-) or die("Error in command line arguments".$!);
-
-if($clearEnv == 0 && $clearNAME == 0 && $clearSurvey == 0) {
- die "ERROR: Specify at least one of environment, NAME and survey to be cleared";
-}
-
-print "Clearing data for date ".$todayString."\n";
-
-if($clearNAME) {
-
- print "Clearing NAME data\n";
-
- #Remove downloaded and unzipped data:
- my $todayFolderPath = $workspacePath."/".$todayString;
-
- #Make a backup of any of todays output, if it exists
- if(-e $todayFolderPath) {
- my $n = 0;
- my $bakFolderPath = "";
- do {
- $bakFolderPath = $todayFolderPath."_bak_".$n;
- $n++;
- } while (-e $bakFolderPath);
-
- dircopy($todayFolderPath, $bakFolderPath);
- }
-
- remove_tree($todayFolderPath);
-
- #Plotting:
- my $plotPath = "/storage/app/Ethiopia-EWS/Plotting/";
-
- my $plotOutputPath = $plotPath."/animation_images/output/NAMEOUTPUT/".$todayString."0000/";
-
- remove_tree($plotOutputPath);
-
- my $csvOutputPath = $plotPath."/output/NAMEOUTPUT/".$todayString."0000/";
-
- remove_tree($csvOutputPath);
-
- #Staging:
- #TODO: use ssh command to nuke stuff on willow
-}
-
-if($clearEnv) {
-
- print "Clearing ENV data\n";
-
- #Remove downloaded and unzipped data:
- my $todayFolderPath = $workspacePath."/ENVIRONMENT_".$todayString;
-
- #Make a backup of any of todays output, if it exists
- if(-e $todayFolderPath) {
- my $n = 0;
- my $bakFolderPath = "";
- do {
- $bakFolderPath = $todayFolderPath."_bak_".$n;
- $n++;
- } while (-e $bakFolderPath);
-
- dircopy($todayFolderPath, $bakFolderPath);
- }
-
- remove_tree($todayFolderPath);
-
- #Plotting:
- my $plotPath = "/storage/app/Ethiopia-EWS/Plotting/";
-
- my $plotOutputPath = $plotPath."/animation_images/output/WheatStemRustOutput/".$todayString."0000/";
-
- remove_tree($plotOutputPath);
-
- $plotOutputPath = $plotPath."/animation_images/output/StripeRustOutput/All_parts/".$todayString."0000/";
-
- remove_tree($plotOutputPath);
-
- my $csvOutputPath = $plotPath."/output/WheatStemRustOutput/".$todayString."0000/";
-
- remove_tree($csvOutputPath);
-
- $csvOutputPath = $plotPath."/output/StripeRustOutput/All_parts/".$todayString."0000/";
-
- remove_tree($csvOutputPath);
-
- $csvOutputPath = $plotPath."/output/StripeRustOutput/part_1/".$todayString."0000/";
-
- remove_tree($csvOutputPath);
-
- $csvOutputPath = $plotPath."/output/StripeRustOutput/part_2/".$todayString."0000/";
-
- remove_tree($csvOutputPath);
-
- $csvOutputPath = $plotPath."/output/StripeRustOutput/part_3/".$todayString."0000/";
-
- remove_tree($csvOutputPath);
-
- $csvOutputPath = $plotPath."/output/StripeRustOutput/part_4/".$todayString."0000/";
-
- remove_tree($csvOutputPath);
-
- $csvOutputPath = $plotPath."/output/StripeRustOutput/part_5/".$todayString."0000/";
-
- remove_tree($csvOutputPath);
-
- $csvOutputPath = $plotPath."/output/StripeRustOutput/part_6/".$todayString."0000/";
-
- remove_tree($csvOutputPath);
-
-
- #Staging:
- #TODO: use ssh command to nuke stuff on willow
-}
-
-if($clearSurvey) {
-
- print "Clearing Survey data\n";
-
- #Remove downloaded and unzipped data:
- my $todayFolderPath = $workspacePath."/SURVEYDATA_".$todayString;
-
- #Make a backup of any of todays output, if it exists
- if(-e $todayFolderPath) {
- my $n = 0;
- my $bakFolderPath = "";
- do {
- $bakFolderPath = $todayFolderPath."_bak_".$n;
- $n++;
- } while (-e $bakFolderPath);
-
- dircopy($todayFolderPath, $bakFolderPath);
- }
-
- remove_tree($todayFolderPath);
-
- #TODO: Nuke any intermediate clustering script output?
-
- #Staging:
- #TODO: use ssh command to nuke stuff on willow
-}
--
GitLab