From ddc87ef8b04009d29477887c9c121c890b42a173 Mon Sep 17 00:00:00 2001
From: Vitor Trovisco <vmt25@cam.ac.uk>
Date: Thu, 23 Sep 2021 10:47:49 +0000
Subject: [PATCH] weekly_export.sh - testing parallel copying the export json
file to a couple of different folders
---
weekly_export.sh | 23 +++++++++++++----------
1 file changed, 13 insertions(+), 10 deletions(-)
diff --git a/weekly_export.sh b/weekly_export.sh
index 75b51d5..5c7840e 100755
--- a/weekly_export.sh
+++ b/weekly_export.sh
@@ -2,7 +2,7 @@
# This script exports data from canto - to be set to run automatically on Wednesdays noon.
-# Define canto-space root
+# Define canto-space root in flybase-vm
CANTOSPACE="/data/export/canto-space"
# Define logs folder inside canto-space
@@ -11,14 +11,12 @@ LOGS="logs"
# Define log file
LOGFILE="${CANTOSPACE}/${LOGS}/canto_weekly_export.log"
+# Define path to staging folder in flybase-vm
+STAGING_CANTOEXPORT="/data/export/staging/canto_export_json"
+
# Redirect all output to logfile
exec &>> "${LOGFILE}"
-# Create archive directory if required
-EXPORT_DIR="${CANTOSPACE}/import_export"
-
-EXPORT_FILENAME=canto_server_export_latest.json
-
# Function to log messages with date stamp
# e.g. log "Canto script completed successfully" generates something like 2020-05-20 10:24:37: Canto script completed successfully
@@ -27,7 +25,7 @@ function log () {
/bin/echo "${DATESTAMP}: ${1}"
}
-# Create archive directory if required
+# Define export directory filepath
EXPORT_DIR="${CANTOSPACE}/import_export"
EXPORT_FILENAME=canto_server_export_latest.json
@@ -50,10 +48,15 @@ if ./canto/script/canto_docker --non-interactive ./script/canto_export.pl canto
EXPORT_FILENAME_ARCHIVE="canto_server_export_$(date +"%Y-%m-%d-%T").json"
- # Copying canto_server_export_latest.json to archive
- log "Copying ${EXPORT_FILENAME} to ${ARCHIVE}/${EXPORT_FILENAME_ARCHIVE}"
+ ## Copying canto_server_export_latest.json to local canto-space archive and flybase-vm staging folder
+ #log "Archiving ${EXPORT_FILENAME} to ${ARCHIVE}/${EXPORT_FILENAME_ARCHIVE} and ${STAGING_CANTOEXPORT}/${EXPORT_FILENAME_ARCHIVE}"
+
+ #parallel cp -v "${EXPORT_DIR}/${EXPORT_FILENAME}" ::: "${ARCHIVE}/${EXPORT_FILENAME_ARCHIVE}" "${STAGING_CANTOEXPORT}/${EXPORT_FILENAME_ARCHIVE}"
+
+ # Test parallel copying of canto_server_export_latest.json to local canto-space archive and (for testing purposes) to the origin folder
+ log "Archiving ${EXPORT_FILENAME} to ${ARCHIVE}/${EXPORT_FILENAME_ARCHIVE} and ${EXPORT_DIR}/${EXPORT_FILENAME_ARCHIVE}"
- cp "${EXPORT_DIR}/${EXPORT_FILENAME}" "${ARCHIVE}/${EXPORT_FILENAME_ARCHIVE}"
+ parallel cp -v "${EXPORT_DIR}/${EXPORT_FILENAME}" ::: "${ARCHIVE}/${EXPORT_FILENAME_ARCHIVE}" "${EXPORT_DIR}/${EXPORT_FILENAME_ARCHIVE}"
log "Finished"
--
GitLab