More changes

- Added function check of db in canto_done file has changed since last run. If not then restart Canto and exit.
- Tidied up code ordering to make it more logical
- Create data folder to store some files to keep things tidy. It be sensible to put fbrf_input_list.tsv in there as well.

weekly_routine.sh

@@ -21,8 +21,6 @@ function log () {
     /bin/echo "${DATESTAMP}: ${1}"
 }
 
-ARCHIVE="${CANTOSPACE}/archive"
-
 # copy/update of ontologies from /data/export/curfiles/ontologies/trunk/ into 'canto-space/import_export'
 # GM comment: I don't really know how rsync works so I didn't know which options to choose (so didn't try to!) and the syntax may not be correct, but the first path should be OK if Canto is installed on the current vm.
 # VT comment: the second path should be to /canto-space/import_export/
@@ -85,44 +83,41 @@ function restart_canto()
 
     log "Finished"
 
+    exit
+
 }
 
-log "Starting"
+function check_if_canto_restart_required()
+{
+    # Compare contents of previous canto_done to current.
+    # If no change then reload canto only
 
-for FILE in "fly_anatomy.obo" "flybase_controlled_vocabulary.obo" "fly_development.obo" "go-basic.obo"; do
-    update_obo_file ${FILE}
-done
+    # if no previous canto_done then create one
+    if [[ ! -e "${PREVMARKERFILE}" ]]; then
+        cp "${MARKERFILE}" "${PREVMARKERFILE}"
+    else
+        # Get hash of MARKERFILE
+        MARKERHASH=$(sha1sum "${MARKERFILE}")
 
-# Create archive directory if required
- if [[ ! -d "${ARCHIVE}" ]]; then
-    log "Creating ${ARCHIVE}"
-    mkdir "${ARCHIVE}"
-fi
+        # Get hash of PREVMARKERFILE
+        PREVHASH=$(sha1sum "${PREVMARKERFILE}")
 
-#replace merged ontology and reload all ontologies to Canto
-# VT comment: Ideally, add the following 'if' routine . If hard to implement, remove the 'if' routine and make the three commands run by default
-# VT comments2: the 'if' loop here may save significant time - takes 20min on my local vm
-# JWRN comment: how do we know an ontology has been changed? Is there a piece of information we can write out and read back in?
-# VT comments2: response to JWRN's question above: as these files will not be necessarily updated every week, the simpler way would be to check the timestamp. If no changes in the last, say, 24h, then you can assume no change has been made
-# JWRN comment: may be easier to just update whatever and improve in time.
-# JWRN comment: commenting out test for the moment.
+        # Make copy of marker file for comparison next time
+        cp "${MARKERFILE}" "${PREVMARKERFILE}"
 
-# If DATACHANGED was set to YES in function above then reload Canto data
-if [[ "${DATACHANGED}" == "YES" ]]; then
+        if "${MARKERHASH}" == "${PREVHASH}"; then
+            echo "No change in database. No further processing required."
 
-    log "Reloading ontologies into Canto"
+            # Restart canto
+            restart_canto
+        else
+            echo "${MARKERFILE} has changed. Processing data..."
 
-    # redo/replace merged FBbt-GO.obo ontology
-    sh ./FBbt-GO_routine/FBbt-GO_routine.sh
 
-    # replace extension_config.tsv
-    sh ./extension_config-Rscript/list-to-extension_config.sh
+    fi
 
-    # reload the ontologies	and extension configuration
-    # JWRN comment: I suspect the sudo here is superfluous as the script is running as root so removed
-    ./canto/script/canto_docker ./script/canto_load.pl --process-extension-config --ontology  /import_export/FBbt-GO.obo --ontology    /import_export/fly_development.obo --ontology   /import_export/flybase_controlled_vocabulary.obo
-fi
 
+}
 
 # Function to retry command until sucessful with max number of attempts
 function retry {
@@ -161,22 +156,41 @@ function retry {
                 # Restart canto
                 restart_canto
 
-                # Exit script
-                exit 1
             fi
         }
     done
 }
 
-# Set filename of file to pull from 
-MARKERFILE="./canto_done"
+log "Starting"
+
+# Create archive directory if required
+ARCHIVE="${CANTOSPACE}/archive"
+
+ if [[ ! -d "${ARCHIVE}" ]]; then
+    log "Creating ${ARCHIVE}"
+    mkdir "${ARCHIVE}"
+fi
+
+# Create data directory if required
+DATA="${CANTOSPACE}/data"
+
+ if [[ ! -d "${DATA}" ]]; then
+    log "Creating ${DATA}"
+    mkdir "${DATA}"
+fi
+
+# Set filename of file to pull from upstream server
+MARKERFILE="${DATA}/canto_done"
+
+# Set filename of file from previous run
+PREVMARKERFILE="${DATA}/canto_done.previous"
 
 # Use retry function to pull marker file from deneb
 retry /usr/bin/scp -q fbadmin@deneb.pdn.cam.ac.uk:instance/canto_done "${MARKERFILE}"
 
 # Get DBNAME from downloaded file
 if [[ -e "${MARKERFILE}" ]]; then
-    DBNAME=$(cat "${MARKERFILE}")
+    DBNAME=$(< "${MARKERFILE}")
 else
     log "${MARKERFILE} does not exist, cannot continue"
     exit 1
@@ -188,6 +202,9 @@ if [[ -z "${DBNAME}" ]]; then
     exit 1
 fi
 
+# If DBNAME hasn't changed then restart canto and quit
+check_if_canto_restart_required
+
 # Transform $DBNAME 
 CANTO_CONFIG="./canto/canto_deploy.yaml"
 log "Inserting ${DBNAME} into ${CANTO_CONFIG}"
@@ -199,6 +216,37 @@ else
     exit 1
 fi
 
+# Loop through files and check if FILE has been updated
+for FILE in "fly_anatomy.obo" "flybase_controlled_vocabulary.obo" "fly_development.obo" "go-basic.obo"; do
+    update_obo_file ${FILE}
+done
+
+
+#replace merged ontology and reload all ontologies to Canto
+# VT comment: Ideally, add the following 'if' routine . If hard to implement, remove the 'if' routine and make the three commands run by default
+# VT comments2: the 'if' loop here may save significant time - takes 20min on my local vm
+# JWRN comment: how do we know an ontology has been changed? Is there a piece of information we can write out and read back in?
+# VT comments2: response to JWRN's question above: as these files will not be necessarily updated every week, the simpler way would be to check the timestamp. If no changes in the last, say, 24h, then you can assume no change has been made
+# JWRN comment: may be easier to just update whatever and improve in time.
+# JWRN comment: commenting out test for the moment.
+
+# If DATACHANGED was set to YES in function above then reload Canto data
+if [[ "${DATACHANGED}" == "YES" ]]; then
+
+    log "Reloading ontologies into Canto"
+
+    # redo/replace merged FBbt-GO.obo ontology
+    sh ./FBbt-GO_routine/FBbt-GO_routine.sh
+
+    # replace extension_config.tsv
+    sh ./extension_config-Rscript/list-to-extension_config.sh
+
+    # reload the ontologies	and extension configuration
+    # JWRN comment: I suspect the sudo here is superfluous as the script is running as root so removed
+    ./canto/script/canto_docker ./script/canto_load.pl --process-extension-config --ontology  /import_export/FBbt-GO.obo --ontology    /import_export/fly_development.obo --ontology   /import_export/flybase_controlled_vocabulary.obo
+fi
+
+
 
 # data import (using Gillian's scripts in the vm - see point 7.d in https://docs.google.com/document/d/19C-J8sJmZb_OSluxyzBWJxUkdR_N4sIpgjHI7u5pp0I/edit)
 # GM comment: the following 'if' command should work
@@ -208,15 +256,18 @@ fi
 # if there are no new papers to add in a particular week, the output file,
 # fbrf_input_list.tsv, will be empty
 
+# Define refs import file
 CURATEDPAPERLISTFILE="./fbrf_input_list.tsv"
+
 log "Creating ${CURATEDPAPERLISTFILE}"
 /usr/bin/perl /data/export/support_scripts/get_fbrfs_to_add_to_canto.pl /data/export/support_scripts/modules_server.cfg > "${CURATEDPAPERLISTFILE}"
 
-JSONIMPORTFILE="/import_export/import-fbrfs.json"
-
 # Test fbrf_input_list.tsv exists and isn't empty
 if [[ ( -e "${CURATEDPAPERLISTFILE}" ) && ( -s "${CURATEDPAPERLISTFILE}" ) ]]; then
 
+    # Define JSON import file
+    JSONIMPORTFILE="/import_export/import-fbrfs.json"
+
     # make the json input file
     log "Creating ${JSONIMPORTFILE}"
     /usr/bin/perl /data/export/support_scripts/canto_json_input_maker.pl /data/export/support_scripts/modules_server.cfg "${CURATEDPAPERLISTFILE}" > ".${JSONIMPORTFILE}"
@@ -263,23 +314,5 @@ if [[ -e ".${JSONIMPORTFILE}" ]]; then
 
 fi
 
-# Restart Canto
+# Restart Canto post processing
 restart_canto
-
-# # Reset cache (restart memcached)
-# if /etc/init.d/memcached restart; then
-#     log "Cache cleared successfully"
-# else
-#     log "Cache clearing failed!"
-# fi
-
-# # Canto restart
-# # Get Container ID from output
-# DOCKERNAME=$(docker ps -a --filter status=running | awk '$2 ~ /pombase\/canto-run/ { print $1 }')
-# if docker restart "${DOCKERNAME}"; then
-#     log "Canto restarted successfully"
-# else
-#     log "Canto failed to restart sucessfully!"
-# fi
-
-# log "Finished"
\ No newline at end of file