From ced34c013657dc9de72cb4ae4ff5835de547e1ab Mon Sep 17 00:00:00 2001
From: Adam Thorn <alt36@cam.ac.uk>
Date: Thu, 27 Jul 2023 11:10:16 +0100
Subject: [PATCH] move prepare scripts out of /etc

These are not config files, and we should not be modifying the package-provided
versions of these files. I'm leaving symlinks behind to make sure we don't break
all our existing backups though!
---
 .../zfs-rsync.d/mysql-dump-script             |  57 +--------
 .../zfs-rsync.d/prepare                       | 108 +-----------------
 .../prepare-ignore-mysql-and-postgres         |  81 +------------
 .../zfs-rsync.d/prepare-nondebian             |  49 +-------
 .../zfs-rsync.d/prepare-redhat                |  48 +-------
 .../prepare-scripts/mysql-dump-script         |  56 +++++++++
 .../prepare-scripts/prepare                   | 107 +++++++++++++++++
 .../prepare-ignore-mysql-and-postgres         |  80 +++++++++++++
 .../prepare-scripts/prepare-nondebian         |  48 ++++++++
 .../prepare-scripts/prepare-redhat            |  47 ++++++++
 10 files changed, 343 insertions(+), 338 deletions(-)
 mode change 100755 => 120000 ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/mysql-dump-script
 mode change 100755 => 120000 ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare
 mode change 100755 => 120000 ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-ignore-mysql-and-postgres
 mode change 100755 => 120000 ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-nondebian
 mode change 100755 => 120000 ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-redhat
 create mode 100755 ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/mysql-dump-script
 create mode 100755 ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare
 create mode 100755 ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-ignore-mysql-and-postgres
 create mode 100755 ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-nondebian
 create mode 100755 ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-redhat

diff --git a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/mysql-dump-script b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/mysql-dump-script
deleted file mode 100755
index 6dc0b61..0000000
--- a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/mysql-dump-script
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/bash
-CONFDIR=/etc/chem-zfs-backup-server/zfs-rsync.d
-SSH="ssh -p ${SSHPORT:-22} -o ConnectTimeout=10"
-
-# to restore a backup of a database taken by this script, begin by 
-# copying mysql-files/$DATABASE/*.gz  to /var/lib/mysql-files/$DATABASE
-# on the mysql server and gunzip them all. Then you might:
-#
-# drop database $DATABASE;
-# create database $DATABASE;
-# for i in /var/lib/mysql-files/$DATABASE/*.sql; do mysql --defaults-file=/etc/mysql/debian.cnf $DATABASE < $i; done
-# for i in /var/lib/mysql-files/$DATABASE/.txt; do mysqlimport --defaults-file=/etc/mysql/debian.cnf $DATABASE $i; done
-#
-# Note: by default mysql is strict about filesystem locations it can export to/import from, and the default is
-# /var/lib/mysql-files . The mysqlimport needs to have the a fully-qualified filepath
-#
-# Also, trigger functions are exported to $tablename.sql, but regular functions end up in the mysql.proc table.
-# Determining the best way to restore those is left as an exercise for the interested reader.
-# Similar comments apply to users, grants etc - see mysql.user, mysql.tables_priv probably
-
-set -xv
-SERVER=$1
-if [ -z $SERVER ] ; then
- echo $0 Server
- exit 1
-fi
-
-(
-$SSH root@$SERVER ls -1 /root/.my.cnf >/dev/null 2>&1
-if [ $? -eq 0 ] ; then
-  MYSQL=mysql
-  MYSQLDUMP=mysqldump
-else
-  MYSQL="mysql --defaults-file=/etc/mysql/debian.cnf"
-  MYSQLDUMP="mysqldump --defaults-file=/etc/mysql/debian.cnf"
-fi
-
-$SSH root@$SERVER "
-set -e
-SQLDIR=/var/lib/mysql-files/
-
-mkdir -p \$SQLDIR
-chown mysql:mysql \$SQLDIR
-
-DBS=\$($MYSQL -NB -e \"show databases\" | grep -vw information_schema | grep -vw performance_schema)
-
-for DB in \$DBS; do
- DB_DIR=\$SQLDIR/\$DB
- mkdir -p \$DB_DIR
- chown mysql:mysql \$DB_DIR
- rm -f \$DB_DIR/*.gz
- $MYSQLDUMP --single-transaction --tab=\$DB_DIR \$DB
- gzip -r \$DB_DIR
-done
-"
-) >/var/log/chem-zfs-backup-server/${SERVER}-mysql-prepare 2>&1
diff --git a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/mysql-dump-script b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/mysql-dump-script
new file mode 120000
index 0000000..8ff0e80
--- /dev/null
+++ b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/mysql-dump-script
@@ -0,0 +1 @@
+/usr/lib/chem-zfs-backup-server/prepare-scripts/mysql-dump-script
\ No newline at end of file
diff --git a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare
deleted file mode 100755
index a4931a4..0000000
--- a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/bin/bash
-# A script which runs, largely on the remote machine, to prepare for backup
-# Jobs to do: 
-# * Exclude files which are only found in packages
-# * List all packages which are installed
-# * (optionally) dump MySQL into a file
-# * (optionally) dump PostgreSQL into a file
-CONFDIR=/etc/chem-zfs-backup-server/zfs-rsync.d
-SSH="ssh -p ${SSHPORT:-22} -o ConnectTimeout=10"
-SCP="scp -P ${SSHPORT:-22}"
-
-set -xv
-
-SERVER=$1
-if [ -z $SERVER ] ; then
- echo $0 Server
- exit 1
-fi
-(
-set -e
-# Has the package status changed since last time we generated the list of files?
-if $SSH root@$SERVER [ /var/lib/dpkg/status -nt /var/adm/backup/package-files ] ; then
- # Need to rebuild package-files
- $SSH root@$SERVER "
-  set -e
-  set -o pipefail
-  umask 077
-  FILELIST=\`mktemp\`
-  CONFLIST=\`mktemp\`
-  DIFFLIST=\'mktemp\'
-  mkdir -p /var/adm/backup
-  # Which packages are installed?
-  dpkg --get-selections | awk ' { print \$1 ; } ' >/var/adm/backup/packages
-  cat /var/lib/dpkg/info/*.list | while read F ; do [ -f \"\$F\" ] && echo \"\$F\" ; done | sort > \$FILELIST
-  awk '/Description:/ { flag = 0 } ; flag == 1 { print \$1 ; }  ; /Conffiles:/ { flag = 1  } ; ' </var/lib/dpkg/status |  sort >\$CONFLIST
-  # ignore the fact that diff exits non-zero when differences are found
-  set +e
-  diff -u \$FILELIST \$CONFLIST  > \$DIFFLIST
-  set -e
-  cat \$DIFFLIST | grep ^-/ | sed s/^-// | tr '\\n' '\\0' | xargs -n1 -0 realpath >/var/adm/backup/package-files
-  rm \$FILELIST \$CONFLIST \$DIFFLIST
- "
- mkdir -p $CONFDIR/$SERVER
- $SCP -q root@$SERVER:/var/adm/backup/package-files $CONFDIR/$SERVER/exclude
-fi
-# Ensure that the excludes file exists, even if we haven't updated it this time
-if [ ! -f $CONFDIR/$SERVER/exclude ] ; then
-  mkdir -p $CONFDIR/$SERVER
- $SCP -q root@$SERVER:/var/adm/backup/package-files $CONFDIR/$SERVER/exclude
-fi
-
-# If we can identify this as a xen VM, add some extra dirs to the exclude list
-# because they include some files built by the kernel package but are not
-# provided by it.
-# systemd-detect-virt is present on all our Ubuntu VMs so is hopefully a 
-# reliable method to use
-# NB this returns non-zero on a non-virtual machine so we don't mind
-# the return value
-set +e
-VIRT_TYPE=$(ssh root@$SERVER systemd-detect-virt 2>/dev/null)
-set -e
-if [ "$VIRT_TYPE" == "xen" ] ; then
-cat<<EOF >> $CONFDIR/$SERVER/exclude
-/lib/modules
-/lib/firmware
-EOF
-fi
-
-if $SSH root@$SERVER id postgres >/dev/null 2>&1 ; then
-  PGVER=$($SSH root@$SERVER pg_lsclusters| awk -F '[. ]+' '/online/ { print $1 }')
-  if [[ -n "$PGVER" ]]; then
-    if (( "$PGVER" >= 12 )) ; then
-      $SSH root@$SERVER "umask 077; su -c 'pg_dumpall --clean' postgres |gzip -9 >/var/adm/backup/postgres" || echo Postgres failed on $SERVER
-    else
-      $SSH root@$SERVER "umask 077; su -c 'pg_dumpall --oids --clean' postgres |gzip -9 >/var/adm/backup/postgres" || echo Postgres failed on $SERVER
-    fi
-  fi
-fi
-
-$SSH root@$SERVER "
-set -e
-# Ensure that things are not readable where they ought not to be
-umask 077
-
-# Save the package configuration info
-which debconf-get-selections >/dev/null 2>&1 || apt-get install debconf-utils
-which rsync >/dev/null 2>&1 || apt-get install rsync
-debconf-get-selections  >/var/adm/backup/debconf
-
-# Dump MySQL, if the root user has access
-set +e
-systemctl is-enabled mysql --quiet 2>/dev/null
-MYSQL_ENABLED=\$?
-set -e
-
-if [ \$MYSQL_ENABLED -eq 0 ] ; then
-  if [ -f /root/.my.cnf ] ; then
-    DEFAULTS_FILE=/root/.my.cnf
-  else
-    DEFAULTS_FILE=/etc/mysql/debian.cnf
-  fi
-  mysqldump  --defaults-file=\$DEFAULTS_FILE --all-databases >/var/adm/backup/mysql
-fi
-
-"
-echo Prepared $SERVER `date`
-) >/var/log/chem-zfs-backup-server/${SERVER}-prepare 2>&1
diff --git a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare
new file mode 120000
index 0000000..d182cd9
--- /dev/null
+++ b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare
@@ -0,0 +1 @@
+/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare
\ No newline at end of file
diff --git a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-ignore-mysql-and-postgres b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-ignore-mysql-and-postgres
deleted file mode 100755
index d7f24bf..0000000
--- a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-ignore-mysql-and-postgres
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/bin/bash
-# A script which runs, largely on the remote machine, to prepare for backup
-# Jobs to do: 
-# * Exclude files which are only found in packages
-# * List all packages which are installed
-# * (optionally) dump PostgreSQL into a file
-
-# NB this is intended to be used in conjunction with a separate mysql backup
-# task. To do this:
-# 1) /usr/lib/chem-zfs-backup-server/new-backup-rsync $HOSTNAME mysql
-# 2) add "--exclude=/var/lib/mysql --exclude=/var/lib/mysql-files" to
-#    to zfs-rsync.d config for $HOSTNAME
-
-CONFDIR=/etc/chem-zfs-backup-server/zfs-rsync.d
-SSH="ssh -p ${SSHPORT:-22} -o ConnectTimeout=10"
-SCP="scp -P ${SSHPORT:-22}"
-#echo $SSH
-set -xv
-SERVER=$1
-if [ -z $SERVER ] ; then
- echo $0 Server
- exit 1
-fi
-(
-# Has the package status changed since last time we generated the list of files?
-if $SSH root@$SERVER [ /var/lib/dpkg/status -nt /var/adm/backup/package-files ] ; then
- # Need to rebuild package-files
- $SSH root@$SERVER "
-  umask 077
-  FILELIST=\`mktemp\`
-  CONFLIST=\`mktemp\`
-  mkdir -p /var/adm/backup
-  # Make logrotate use datestamps
-  if ! grep -q dateext /etc/logrotate.conf ; then sed  -i 's/^include/dateext\ninclude/' /etc/logrotate.conf ; fi
-  # Which packages are installed?
-  dpkg --get-selections | awk ' { print \$1 ; } ' >/var/adm/backup/packages
-  cat /var/lib/dpkg/info/*.list | while read F ; do [ -f \"\$F\" ] && echo \"\$F\" ; done | sort > \$FILELIST
-  awk '/Description:/ { flag = 0 } ; flag == 1 { print \$1 ; }  ; /Conffiles:/ { flag = 1  } ; ' </var/lib/dpkg/status |  sort >\$CONFLIST
-  diff -u \$FILELIST \$CONFLIST | grep ^-/ | sed s/^-// | xargs -n1 realpath >/var/adm/backup/package-files
-  rm \$FILELIST \$CONFLIST
- "
- mkdir -p $CONFDIR/$SERVER
- $SCP -q root@$SERVER:/var/adm/backup/package-files $CONFDIR/$SERVER/exclude
-fi
-# Ensure that the excludes file exists, even if we haven't updated it this time
-if [ ! -f $CONFDIR/$SERVER/exclude ] ; then
-  mkdir -p $CONFDIR/$SERVER
- $SCP -q root@$SERVER:/var/adm/backup/package-files $CONFDIR/$SERVER/exclude
-fi
-
-# If we can identify this as a xen VM, add some extra dirs to the exclude list
-# because they include some files built by the kernel package but are not
-# provided by it.
-# systemd-detect-virt is present on all our Ubuntu VMs so is hopefully a
-# reliable method to use
-VIRT_TYPE=$(ssh root@$SERVER systemd-detect-virt 2>/dev/null)
-if [ "$VIRT_TYPE" == "xen" ] ; then
-cat<<EOF >> $CONFDIR/$SERVER/exclude
-/lib/modules
-/lib/firmware
-EOF
-fi
-
-$SSH root@$SERVER "
-# Ensure that things are not readable where they ought not to be
-umask 077
-
-# Save the package configuration info
-which debconf-get-selections >/dev/null 2>&1 || apt-get install debconf-utils
-which rsync >/dev/null 2>&1 || apt-get install rsync
-debconf-get-selections  >/var/adm/backup/debconf
-
-# Dump LDAP, if we can
-which slapcat >/dev/null 2>&1 && slapcat >/var/adm/backup/ldap
-
-# Ignore exit status of this command
-true
-"
-echo Prepared $SERVER `date`
-) >/var/log/chem-zfs-backup-server/${SERVER}-prepare 2>&1
diff --git a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-ignore-mysql-and-postgres b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-ignore-mysql-and-postgres
new file mode 120000
index 0000000..928c05d
--- /dev/null
+++ b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-ignore-mysql-and-postgres
@@ -0,0 +1 @@
+/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-ignore-mysql-and-postgres
\ No newline at end of file
diff --git a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-nondebian b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-nondebian
deleted file mode 100755
index 5617dc6..0000000
--- a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-nondebian
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/bash
-#set -x
-# A script which runs, largely on the remote machine, to prepare for backup
-# Jobs to do: 
-# * Exclude files which are only found in packages
-# * List all packages which are installed
-# * (optionally) dump MySQL into a file
-# * (optionally) dump PostgreSQL into a file
-SERVER=$1
-if [ -z "$SERVER" ] ; then
- echo $0 Server
- exit 1
-fi
-(
-# Create some necessary files
-mkdir -p /etc/chem-zfs-backup-server/zfs-rsync.d/$SERVER
-touch /etc/chem-zfs-backup-server/zfs-rsync.d/$SERVER/exclude
-ssh root@$SERVER "
- umask 077
- mkdir -p /var/adm/backup
- touch /var/adm/backup/packages
- touch /var/adm/backup/package-files
-"
-mkdir -p /etc/rsnapshot/$SERVER
-scp -q root@$SERVER:/var/adm/backup/package-files /etc/rsnapshot/$SERVER/exclude
-
-ssh root@$SERVER "
- # Ensure that things are not readable where they ought not to be
- umask 077
- touch /var/adm/backup/debconf
-
-# Dump MySQL, if the root user has access
-if [ -f /etc/mysql/my.cnf ] ; then
- SOCKET=`awk ' BEGIN {FLAG=0} (\$0 ~ /^\[/) {FLAG=0} (\$0 ~ /^\[mysqld\]/) {FLAG=1} (\$1=="socket" && FLAG==1) { print \$3 ; } ' </etc/mysql/my.cnf`
- [ -S \$SOCKET ] && which mysqldump >/dev/null 2>&1 && mysqldump  --defaults-file=/etc/mysql/debian.cnf --all-databases >/var/adm/backup/mysql
-fi
-
-# Dump PostgresSQL, if the postgres user has access
-id postgres >/dev/null 2>&1 && [ -d /var/run/postgresql/ ] && [ -x /usr/bin/pg_dumpall ] && ( su -c 'pg_dumpall --oids --clean' postgres >/var/adm/backup/postgres || echo Postgres failed on $SERVER )
-
-# Dump LDAP, if we can
-which slapcat >/dev/null 2>&1 && slapcat >/var/adm/backup/ldap
-
-# Ignore exit status of this command
-true
-"
-echo Prepared $SERVER `date`
-) >/var/log/chem-zfs-backup-server/${SERVER}-prepare 2>&1
diff --git a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-nondebian b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-nondebian
new file mode 120000
index 0000000..6179ded
--- /dev/null
+++ b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-nondebian
@@ -0,0 +1 @@
+/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-nondebian
\ No newline at end of file
diff --git a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-redhat b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-redhat
deleted file mode 100755
index 3fc99a3..0000000
--- a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-redhat
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/bin/bash
-#set -x
-# A script which runs, largely on the remote machine, to prepare for backup
-# Jobs to do: 
-# * Exclude files which are only found in packages
-# * List all packages which are installed
-# * (optionally) dump MySQL into a file
-# * (optionally) dump PostgreSQL into a file
-SERVER=$1
-if [ -z "$SERVER" ] ; then
- echo $0 Server
- exit 1
-fi
-(
-# Create some necessary files
-mkdir -p /etc/chem-zfs-backup-server/zfs-rsync.d/$SERVER
-touch /etc/chem-zfs-backup-server/zfs-rsync.d/$SERVER/exclude
-ssh root@$SERVER "
- umask 077
- mkdir -p /var/adm/backup
- touch /var/adm/backup/packages
- touch /var/adm/backup/package-files
-"
-mkdir -p /etc/rsnapshot/$SERVER
-scp -q root@$SERVER:/var/adm/backup/package-files /etc/rsnapshot/$SERVER/exclude
-
-ssh root@$SERVER "
- # Ensure that things are not readable where they ought not to be
- umask 077
-
-# Dump MySQL, if the root user has access
-if [ -f /etc/my.cnf ] ; then
- SOCKET=`awk ' BEGIN {FLAG=0} (\$0 ~ /^\[/) {FLAG=0} (\$0 ~ /^\[mysqld\]/) {FLAG=1} (\$1=="socket" && FLAG==1) { print \$3 ; } ' </etc/my.cnf`
- [ -S \$SOCKET ] && which mysqldump >/dev/null 2>&1 && mysqldump --all-databases | gzip -c >/var/adm/backup/mysql.gz
-fi
-
-# Dump PostgresSQL, if the postgres user has access
-id postgres >/dev/null 2>&1 && [ -d /var/run/postgresql/ ] && [ -x /usr/bin/pg_dumpall ] && ( su -c 'pg_dumpall --oids --clean' postgres >/var/adm/backup/postgres || echo Postgres failed on $SERVER )
-
-# Dump LDAP, if we can
-which slapcat >/dev/null 2>&1 && slapcat >/var/adm/backup/ldap
-
-# Ignore exit status of this command
-true
-"
-echo Prepared $SERVER `date`
-) >/var/log/chem-zfs-backup-server/${SERVER}-prepare 2>&1
diff --git a/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-redhat b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-redhat
new file mode 120000
index 0000000..4684cda
--- /dev/null
+++ b/ROOT/etc/chem-zfs-backup-server/zfs-rsync.d/prepare-redhat
@@ -0,0 +1 @@
+/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-redhat
\ No newline at end of file
diff --git a/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/mysql-dump-script b/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/mysql-dump-script
new file mode 100755
index 0000000..6dc0b61
--- /dev/null
+++ b/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/mysql-dump-script
@@ -0,0 +1,56 @@
+#!/bin/bash
+CONFDIR=/etc/chem-zfs-backup-server/zfs-rsync.d
+SSH="ssh -p ${SSHPORT:-22} -o ConnectTimeout=10"
+
+# to restore a backup of a database taken by this script, begin by 
+# copying mysql-files/$DATABASE/*.gz  to /var/lib/mysql-files/$DATABASE
+# on the mysql server and gunzip them all. Then you might:
+#
+# drop database $DATABASE;
+# create database $DATABASE;
+# for i in /var/lib/mysql-files/$DATABASE/*.sql; do mysql --defaults-file=/etc/mysql/debian.cnf $DATABASE < $i; done
+# for i in /var/lib/mysql-files/$DATABASE/.txt; do mysqlimport --defaults-file=/etc/mysql/debian.cnf $DATABASE $i; done
+#
+# Note: by default mysql is strict about filesystem locations it can export to/import from, and the default is
+# /var/lib/mysql-files . The mysqlimport needs to have the a fully-qualified filepath
+#
+# Also, trigger functions are exported to $tablename.sql, but regular functions end up in the mysql.proc table.
+# Determining the best way to restore those is left as an exercise for the interested reader.
+# Similar comments apply to users, grants etc - see mysql.user, mysql.tables_priv probably
+
+set -xv
+SERVER=$1
+if [ -z $SERVER ] ; then
+ echo $0 Server
+ exit 1
+fi
+
+(
+$SSH root@$SERVER ls -1 /root/.my.cnf >/dev/null 2>&1
+if [ $? -eq 0 ] ; then
+  MYSQL=mysql
+  MYSQLDUMP=mysqldump
+else
+  MYSQL="mysql --defaults-file=/etc/mysql/debian.cnf"
+  MYSQLDUMP="mysqldump --defaults-file=/etc/mysql/debian.cnf"
+fi
+
+$SSH root@$SERVER "
+set -e
+SQLDIR=/var/lib/mysql-files/
+
+mkdir -p \$SQLDIR
+chown mysql:mysql \$SQLDIR
+
+DBS=\$($MYSQL -NB -e \"show databases\" | grep -vw information_schema | grep -vw performance_schema)
+
+for DB in \$DBS; do
+ DB_DIR=\$SQLDIR/\$DB
+ mkdir -p \$DB_DIR
+ chown mysql:mysql \$DB_DIR
+ rm -f \$DB_DIR/*.gz
+ $MYSQLDUMP --single-transaction --tab=\$DB_DIR \$DB
+ gzip -r \$DB_DIR
+done
+"
+) >/var/log/chem-zfs-backup-server/${SERVER}-mysql-prepare 2>&1
diff --git a/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare b/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare
new file mode 100755
index 0000000..a4931a4
--- /dev/null
+++ b/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare
@@ -0,0 +1,107 @@
+#!/bin/bash
+# A script which runs, largely on the remote machine, to prepare for backup
+# Jobs to do: 
+# * Exclude files which are only found in packages
+# * List all packages which are installed
+# * (optionally) dump MySQL into a file
+# * (optionally) dump PostgreSQL into a file
+CONFDIR=/etc/chem-zfs-backup-server/zfs-rsync.d
+SSH="ssh -p ${SSHPORT:-22} -o ConnectTimeout=10"
+SCP="scp -P ${SSHPORT:-22}"
+
+set -xv
+
+SERVER=$1
+if [ -z $SERVER ] ; then
+ echo $0 Server
+ exit 1
+fi
+(
+set -e
+# Has the package status changed since last time we generated the list of files?
+if $SSH root@$SERVER [ /var/lib/dpkg/status -nt /var/adm/backup/package-files ] ; then
+ # Need to rebuild package-files
+ $SSH root@$SERVER "
+  set -e
+  set -o pipefail
+  umask 077
+  FILELIST=\`mktemp\`
+  CONFLIST=\`mktemp\`
+  DIFFLIST=\'mktemp\'
+  mkdir -p /var/adm/backup
+  # Which packages are installed?
+  dpkg --get-selections | awk ' { print \$1 ; } ' >/var/adm/backup/packages
+  cat /var/lib/dpkg/info/*.list | while read F ; do [ -f \"\$F\" ] && echo \"\$F\" ; done | sort > \$FILELIST
+  awk '/Description:/ { flag = 0 } ; flag == 1 { print \$1 ; }  ; /Conffiles:/ { flag = 1  } ; ' </var/lib/dpkg/status |  sort >\$CONFLIST
+  # ignore the fact that diff exits non-zero when differences are found
+  set +e
+  diff -u \$FILELIST \$CONFLIST  > \$DIFFLIST
+  set -e
+  cat \$DIFFLIST | grep ^-/ | sed s/^-// | tr '\\n' '\\0' | xargs -n1 -0 realpath >/var/adm/backup/package-files
+  rm \$FILELIST \$CONFLIST \$DIFFLIST
+ "
+ mkdir -p $CONFDIR/$SERVER
+ $SCP -q root@$SERVER:/var/adm/backup/package-files $CONFDIR/$SERVER/exclude
+fi
+# Ensure that the excludes file exists, even if we haven't updated it this time
+if [ ! -f $CONFDIR/$SERVER/exclude ] ; then
+  mkdir -p $CONFDIR/$SERVER
+ $SCP -q root@$SERVER:/var/adm/backup/package-files $CONFDIR/$SERVER/exclude
+fi
+
+# If we can identify this as a xen VM, add some extra dirs to the exclude list
+# because they include some files built by the kernel package but are not
+# provided by it.
+# systemd-detect-virt is present on all our Ubuntu VMs so is hopefully a 
+# reliable method to use
+# NB this returns non-zero on a non-virtual machine so we don't mind
+# the return value
+set +e
+VIRT_TYPE=$(ssh root@$SERVER systemd-detect-virt 2>/dev/null)
+set -e
+if [ "$VIRT_TYPE" == "xen" ] ; then
+cat<<EOF >> $CONFDIR/$SERVER/exclude
+/lib/modules
+/lib/firmware
+EOF
+fi
+
+if $SSH root@$SERVER id postgres >/dev/null 2>&1 ; then
+  PGVER=$($SSH root@$SERVER pg_lsclusters| awk -F '[. ]+' '/online/ { print $1 }')
+  if [[ -n "$PGVER" ]]; then
+    if (( "$PGVER" >= 12 )) ; then
+      $SSH root@$SERVER "umask 077; su -c 'pg_dumpall --clean' postgres |gzip -9 >/var/adm/backup/postgres" || echo Postgres failed on $SERVER
+    else
+      $SSH root@$SERVER "umask 077; su -c 'pg_dumpall --oids --clean' postgres |gzip -9 >/var/adm/backup/postgres" || echo Postgres failed on $SERVER
+    fi
+  fi
+fi
+
+$SSH root@$SERVER "
+set -e
+# Ensure that things are not readable where they ought not to be
+umask 077
+
+# Save the package configuration info
+which debconf-get-selections >/dev/null 2>&1 || apt-get install debconf-utils
+which rsync >/dev/null 2>&1 || apt-get install rsync
+debconf-get-selections  >/var/adm/backup/debconf
+
+# Dump MySQL, if the root user has access
+set +e
+systemctl is-enabled mysql --quiet 2>/dev/null
+MYSQL_ENABLED=\$?
+set -e
+
+if [ \$MYSQL_ENABLED -eq 0 ] ; then
+  if [ -f /root/.my.cnf ] ; then
+    DEFAULTS_FILE=/root/.my.cnf
+  else
+    DEFAULTS_FILE=/etc/mysql/debian.cnf
+  fi
+  mysqldump  --defaults-file=\$DEFAULTS_FILE --all-databases >/var/adm/backup/mysql
+fi
+
+"
+echo Prepared $SERVER `date`
+) >/var/log/chem-zfs-backup-server/${SERVER}-prepare 2>&1
diff --git a/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-ignore-mysql-and-postgres b/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-ignore-mysql-and-postgres
new file mode 100755
index 0000000..d7f24bf
--- /dev/null
+++ b/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-ignore-mysql-and-postgres
@@ -0,0 +1,80 @@
+#!/bin/bash
+# A script which runs, largely on the remote machine, to prepare for backup
+# Jobs to do: 
+# * Exclude files which are only found in packages
+# * List all packages which are installed
+# * (optionally) dump PostgreSQL into a file
+
+# NB this is intended to be used in conjunction with a separate mysql backup
+# task. To do this:
+# 1) /usr/lib/chem-zfs-backup-server/new-backup-rsync $HOSTNAME mysql
+# 2) add "--exclude=/var/lib/mysql --exclude=/var/lib/mysql-files" to
+#    to zfs-rsync.d config for $HOSTNAME
+
+CONFDIR=/etc/chem-zfs-backup-server/zfs-rsync.d
+SSH="ssh -p ${SSHPORT:-22} -o ConnectTimeout=10"
+SCP="scp -P ${SSHPORT:-22}"
+#echo $SSH
+set -xv
+SERVER=$1
+if [ -z $SERVER ] ; then
+ echo $0 Server
+ exit 1
+fi
+(
+# Has the package status changed since last time we generated the list of files?
+if $SSH root@$SERVER [ /var/lib/dpkg/status -nt /var/adm/backup/package-files ] ; then
+ # Need to rebuild package-files
+ $SSH root@$SERVER "
+  umask 077
+  FILELIST=\`mktemp\`
+  CONFLIST=\`mktemp\`
+  mkdir -p /var/adm/backup
+  # Make logrotate use datestamps
+  if ! grep -q dateext /etc/logrotate.conf ; then sed  -i 's/^include/dateext\ninclude/' /etc/logrotate.conf ; fi
+  # Which packages are installed?
+  dpkg --get-selections | awk ' { print \$1 ; } ' >/var/adm/backup/packages
+  cat /var/lib/dpkg/info/*.list | while read F ; do [ -f \"\$F\" ] && echo \"\$F\" ; done | sort > \$FILELIST
+  awk '/Description:/ { flag = 0 } ; flag == 1 { print \$1 ; }  ; /Conffiles:/ { flag = 1  } ; ' </var/lib/dpkg/status |  sort >\$CONFLIST
+  diff -u \$FILELIST \$CONFLIST | grep ^-/ | sed s/^-// | xargs -n1 realpath >/var/adm/backup/package-files
+  rm \$FILELIST \$CONFLIST
+ "
+ mkdir -p $CONFDIR/$SERVER
+ $SCP -q root@$SERVER:/var/adm/backup/package-files $CONFDIR/$SERVER/exclude
+fi
+# Ensure that the excludes file exists, even if we haven't updated it this time
+if [ ! -f $CONFDIR/$SERVER/exclude ] ; then
+  mkdir -p $CONFDIR/$SERVER
+ $SCP -q root@$SERVER:/var/adm/backup/package-files $CONFDIR/$SERVER/exclude
+fi
+
+# If we can identify this as a xen VM, add some extra dirs to the exclude list
+# because they include some files built by the kernel package but are not
+# provided by it.
+# systemd-detect-virt is present on all our Ubuntu VMs so is hopefully a
+# reliable method to use
+VIRT_TYPE=$(ssh root@$SERVER systemd-detect-virt 2>/dev/null)
+if [ "$VIRT_TYPE" == "xen" ] ; then
+cat<<EOF >> $CONFDIR/$SERVER/exclude
+/lib/modules
+/lib/firmware
+EOF
+fi
+
+$SSH root@$SERVER "
+# Ensure that things are not readable where they ought not to be
+umask 077
+
+# Save the package configuration info
+which debconf-get-selections >/dev/null 2>&1 || apt-get install debconf-utils
+which rsync >/dev/null 2>&1 || apt-get install rsync
+debconf-get-selections  >/var/adm/backup/debconf
+
+# Dump LDAP, if we can
+which slapcat >/dev/null 2>&1 && slapcat >/var/adm/backup/ldap
+
+# Ignore exit status of this command
+true
+"
+echo Prepared $SERVER `date`
+) >/var/log/chem-zfs-backup-server/${SERVER}-prepare 2>&1
diff --git a/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-nondebian b/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-nondebian
new file mode 100755
index 0000000..5617dc6
--- /dev/null
+++ b/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-nondebian
@@ -0,0 +1,48 @@
+#!/bin/bash
+#set -x
+# A script which runs, largely on the remote machine, to prepare for backup
+# Jobs to do: 
+# * Exclude files which are only found in packages
+# * List all packages which are installed
+# * (optionally) dump MySQL into a file
+# * (optionally) dump PostgreSQL into a file
+SERVER=$1
+if [ -z "$SERVER" ] ; then
+ echo $0 Server
+ exit 1
+fi
+(
+# Create some necessary files
+mkdir -p /etc/chem-zfs-backup-server/zfs-rsync.d/$SERVER
+touch /etc/chem-zfs-backup-server/zfs-rsync.d/$SERVER/exclude
+ssh root@$SERVER "
+ umask 077
+ mkdir -p /var/adm/backup
+ touch /var/adm/backup/packages
+ touch /var/adm/backup/package-files
+"
+mkdir -p /etc/rsnapshot/$SERVER
+scp -q root@$SERVER:/var/adm/backup/package-files /etc/rsnapshot/$SERVER/exclude
+
+ssh root@$SERVER "
+ # Ensure that things are not readable where they ought not to be
+ umask 077
+ touch /var/adm/backup/debconf
+
+# Dump MySQL, if the root user has access
+if [ -f /etc/mysql/my.cnf ] ; then
+ SOCKET=`awk ' BEGIN {FLAG=0} (\$0 ~ /^\[/) {FLAG=0} (\$0 ~ /^\[mysqld\]/) {FLAG=1} (\$1=="socket" && FLAG==1) { print \$3 ; } ' </etc/mysql/my.cnf`
+ [ -S \$SOCKET ] && which mysqldump >/dev/null 2>&1 && mysqldump  --defaults-file=/etc/mysql/debian.cnf --all-databases >/var/adm/backup/mysql
+fi
+
+# Dump PostgresSQL, if the postgres user has access
+id postgres >/dev/null 2>&1 && [ -d /var/run/postgresql/ ] && [ -x /usr/bin/pg_dumpall ] && ( su -c 'pg_dumpall --oids --clean' postgres >/var/adm/backup/postgres || echo Postgres failed on $SERVER )
+
+# Dump LDAP, if we can
+which slapcat >/dev/null 2>&1 && slapcat >/var/adm/backup/ldap
+
+# Ignore exit status of this command
+true
+"
+echo Prepared $SERVER `date`
+) >/var/log/chem-zfs-backup-server/${SERVER}-prepare 2>&1
diff --git a/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-redhat b/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-redhat
new file mode 100755
index 0000000..3fc99a3
--- /dev/null
+++ b/ROOT/usr/lib/chem-zfs-backup-server/prepare-scripts/prepare-redhat
@@ -0,0 +1,47 @@
+#!/bin/bash
+#set -x
+# A script which runs, largely on the remote machine, to prepare for backup
+# Jobs to do: 
+# * Exclude files which are only found in packages
+# * List all packages which are installed
+# * (optionally) dump MySQL into a file
+# * (optionally) dump PostgreSQL into a file
+SERVER=$1
+if [ -z "$SERVER" ] ; then
+ echo $0 Server
+ exit 1
+fi
+(
+# Create some necessary files
+mkdir -p /etc/chem-zfs-backup-server/zfs-rsync.d/$SERVER
+touch /etc/chem-zfs-backup-server/zfs-rsync.d/$SERVER/exclude
+ssh root@$SERVER "
+ umask 077
+ mkdir -p /var/adm/backup
+ touch /var/adm/backup/packages
+ touch /var/adm/backup/package-files
+"
+mkdir -p /etc/rsnapshot/$SERVER
+scp -q root@$SERVER:/var/adm/backup/package-files /etc/rsnapshot/$SERVER/exclude
+
+ssh root@$SERVER "
+ # Ensure that things are not readable where they ought not to be
+ umask 077
+
+# Dump MySQL, if the root user has access
+if [ -f /etc/my.cnf ] ; then
+ SOCKET=`awk ' BEGIN {FLAG=0} (\$0 ~ /^\[/) {FLAG=0} (\$0 ~ /^\[mysqld\]/) {FLAG=1} (\$1=="socket" && FLAG==1) { print \$3 ; } ' </etc/my.cnf`
+ [ -S \$SOCKET ] && which mysqldump >/dev/null 2>&1 && mysqldump --all-databases | gzip -c >/var/adm/backup/mysql.gz
+fi
+
+# Dump PostgresSQL, if the postgres user has access
+id postgres >/dev/null 2>&1 && [ -d /var/run/postgresql/ ] && [ -x /usr/bin/pg_dumpall ] && ( su -c 'pg_dumpall --oids --clean' postgres >/var/adm/backup/postgres || echo Postgres failed on $SERVER )
+
+# Dump LDAP, if we can
+which slapcat >/dev/null 2>&1 && slapcat >/var/adm/backup/ldap
+
+# Ignore exit status of this command
+true
+"
+echo Prepared $SERVER `date`
+) >/var/log/chem-zfs-backup-server/${SERVER}-prepare 2>&1
-- 
GitLab