weekly_export.sh - testing parallel copying the export json file to a couple of different folders

weekly_export.sh

@@ -2,7 +2,7 @@
 
 # This script exports data from canto - to be set to run automatically on Wednesdays noon.
 
-# Define canto-space root
+# Define canto-space root in flybase-vm
 CANTOSPACE="/data/export/canto-space"
 
 # Define logs folder inside canto-space
@@ -11,14 +11,12 @@ LOGS="logs"
 # Define log file
 LOGFILE="${CANTOSPACE}/${LOGS}/canto_weekly_export.log"
 
+# Define path to staging folder in flybase-vm
+STAGING_CANTOEXPORT="/data/export/staging/canto_export_json"
+
 # Redirect all output to logfile
 exec &>> "${LOGFILE}"
 
-# Create archive directory if required
-EXPORT_DIR="${CANTOSPACE}/import_export"
-
-EXPORT_FILENAME=canto_server_export_latest.json
-
 # Function to log messages with date stamp
 # e.g. log "Canto script completed successfully" generates something like 2020-05-20 10:24:37: Canto script completed successfully
 
@@ -27,7 +25,7 @@ function log () {
     /bin/echo "${DATESTAMP}: ${1}"
 }
 
-# Create archive directory if required
+# Define export directory filepath
 EXPORT_DIR="${CANTOSPACE}/import_export"
 
 EXPORT_FILENAME=canto_server_export_latest.json
@@ -50,10 +48,15 @@ if ./canto/script/canto_docker  --non-interactive ./script/canto_export.pl canto
 
 		EXPORT_FILENAME_ARCHIVE="canto_server_export_$(date +"%Y-%m-%d-%T").json"
 
-		# Copying canto_server_export_latest.json to archive
-		log "Copying ${EXPORT_FILENAME} to ${ARCHIVE}/${EXPORT_FILENAME_ARCHIVE}"
+		## Copying canto_server_export_latest.json to local canto-space archive and flybase-vm staging folder
+		#log "Archiving ${EXPORT_FILENAME} to ${ARCHIVE}/${EXPORT_FILENAME_ARCHIVE} and ${STAGING_CANTOEXPORT}/${EXPORT_FILENAME_ARCHIVE}"
+
+		#parallel cp -v "${EXPORT_DIR}/${EXPORT_FILENAME}" ::: "${ARCHIVE}/${EXPORT_FILENAME_ARCHIVE}" "${STAGING_CANTOEXPORT}/${EXPORT_FILENAME_ARCHIVE}"
+		
+		# Test parallel copying of canto_server_export_latest.json to local canto-space archive and (for testing purposes) to the origin folder
+		log "Archiving ${EXPORT_FILENAME} to ${ARCHIVE}/${EXPORT_FILENAME_ARCHIVE} and ${EXPORT_DIR}/${EXPORT_FILENAME_ARCHIVE}"
 
-		cp "${EXPORT_DIR}/${EXPORT_FILENAME}" "${ARCHIVE}/${EXPORT_FILENAME_ARCHIVE}"
+		parallel cp -v "${EXPORT_DIR}/${EXPORT_FILENAME}" ::: "${ARCHIVE}/${EXPORT_FILENAME_ARCHIVE}" "${EXPORT_DIR}/${EXPORT_FILENAME_ARCHIVE}"
 	
 		log "Finished"