From 0746d923d047cab0b4ba3f783608782cdd592ac5 Mon Sep 17 00:00:00 2001
From: Jake Smith <jws52@cam.ac.uk>
Date: Mon, 4 Jan 2021 15:34:06 +0000
Subject: [PATCH] Hard-coded check of length of exported survey data as a basic
check of ODK Briefcase for robustness (in particular for exports from
Ethiopia). Currently, 2715 is an expected minimum for Ethiopia. Any small
increases in coming weeks are unlikely to lead to not catching an incomplete
export. This check is not important for SouthAsia data export which has been
reliable. Because of the large data size, this 2715 limit is almost
ineffective for South Asia anyway.
---
SurveyDataProcessor.pl | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/SurveyDataProcessor.pl b/SurveyDataProcessor.pl
index 3fc36f3..e55064f 100755
--- a/SurveyDataProcessor.pl
+++ b/SurveyDataProcessor.pl
@@ -432,6 +432,15 @@ unless($serverFailure || $debugNoDownload) {
#handleError("ERROR: Unable to obtain the survey data from ".$serverName);
}
+ my $csvLength = `cat ${csvOutputDir}/${csvFileName} | wc -l`;
+
+ print("DEBUG: CSV length is ".$csvLength."\n");
+
+ if( $csvLength < 2715) {
+ print("Downloaded csv file appears to be too short.\n");
+ fcopy("${csvOutputDir}/${csvFileName}","${csvOutputDir}/SurveyDataShortDownload.csv") or handleError("ERROR: Failed to copy the bad survey file");
+ $serverFailure = 1;
+ }
}
if ($serverFailure || $debugNoDownload) {
@@ -443,7 +452,7 @@ if ($serverFailure || $debugNoDownload) {
# TODO: Enable lookback for historical dates
my $epoc = time();#a bit fragile if we've gone into a new day since the start of the script (highly unlikely as script takes about a second to execute), but we're looking back 7 days, and looking at "today" again and only going back 6 is still fine
- my $maxLookbackDays = 14;
+ my $maxLookbackDays = 21;
my $staleDownloadDataWarningString = "";
--
GitLab