Backing up files and databases

The files below are what I use to back up my machines, it’s quick, simple, and effective – I’ve used this for years to recover a lot of data, including bare-metal on more than one occasion. The files listed toward the bottom (Summary, Backups, and Filelist) are generated and don’t need to exist beforehand.

/Backup/SiteWideBackup, executable (chmod a+x), this script is called by a cron job, it calls the other scripts that do the actual backups.

#!/bin/bash
# make sure we're running as root
  if (( `id -u` != 0 )); then {
  echo "Sorry, must be root. Exiting..."; exit;
  } fi;
 
# First, make a backup of all of the MySQL Databases
/Backup/MySQL/MySQLBackup
 
# Finally, backup everything else;
# Run a backup for every .conf file in the /Backup directory
  cd /Backup
  for Script in *.conf
    do ./Snapshot $Script
  done
# And send an e-mail with the list of Summaries;
cat /Backup/*/Summary

/Backup/MySQL/MySQLBackup, this backs up all of MySQL.

#!/bin/bash
# This script is an attempt to make good backups of all of the databases within MySQL
# SRJ 2017-05-06
#
#
# First step, create a user solely for executing backups with;
# MariaDB [(none)]> GRANT LOCK TABLES, SELECT ON *.* TO 'MySQLBackup'@'localhost' IDENTIFIED BY 'MeCG57*2F&m(dP';
# Query OK, 0 rows affected (0.00 sec)
# 
# MariaDB [(none)]> flush privileges;
# Query OK, 0 rows affected (0.00 sec)
# 
# MariaDB [(none)]> quit;
# Bye
 
cd /Backup/MySQL
 
Date=$(date +"%Y-%m-%d-%H-%M")
User='MySQLBackup'
Pass='MeCG57*2F&m(dP'
MySQL=/usr/bin/mysql
MySQLDump=/usr/bin/mysqldump
 
Databases=$(${MySQL} --user=${User} -p${Pass} -e "SHOW DATABASES;" | grep -Ev "(Database|information_schema|performance_schema)")
 
# This creates a dated file for each db, these are text files that can be restored with mysql. 
for db in ${Databases} ; do
  ${MySQLDump} --force --opt --user=${User} -p${Pass} --databases "${db}" > ${Date}-"${db}".sql
done
 
# This last line deletes files older than two weeks - touching a file refreshes that clock.
find . -type f -name '*.sql' -atime +14 -exec rm -f {} \;

/Backup/01-Root.conf, this configuration file defines the backups for a partition. The rsync application shouldn’t cross filesystem boundaries because that would break the incremental nature of this backup.

# =-=-=-=-=-=- File locations and variables.  -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
# The physical device where backups will exist.
	MOUNT_DEVICE="LABEL=Backup";
# Where to mount the physical device.
	MOUNT_DIR=/Backup;
# Where to write the backups
	SNAPSHOT_RW=/Backup/01-Root;
# What directory to back up, begins and ends in /
	SNAP_DIR="/";
# Where to store the list of files and directories that we backed up.
        FILELIST=$SNAPSHOT_RW/Filelist
# Summary file, a short list of actions taken by rsync to complete the backup.
        SUMMARY=$SNAPSHOT_RW/Summary
# Stuff not to back up, there is always stuff to not back up!
	EXCLUDES=$SNAPSHOT_RW/Excludes;
# How many snapshot backups do we want to keep? Must be at least 2!
	NUM_SNAP=10;

Snapshot, this is the script that backs up regular files and folders.

#!/bin/bash
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
# 2017-01-06-SRJ
# For information on how this script works, and what it does, read the 
# file README in this folder
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
#
# suggestion from H. Milz: avoid accidental use of $PATH
unset PATH
#
# =-=-=-=-  System commands used by this script. -=-=-=-=-=-=-=-=
   ID=/usr/bin/id; ECHO=/bin/echo; MOUNT=/bin/mount; RM=/bin/rm; 
   MV=/bin/mv; CP=/bin/cp; TOUCH=/bin/touch; RSYNC=/usr/bin/rsync;
   DATE=/bin/date; WC=/usr/bin/wc; CAT=/bin/cat; TAIL=/usr/bin/tail;
   HEAD=/usr/bin/head; GREP=/bin/grep; CHMOD=/bin/chmod
 
# =-=-=-=-=-=-=- Make sure we're running as root -=-=-=-=-=-=-=-=-=
  if (( `$ID -u` != 0 )); then { 
  $ECHO "Sorry, must be root. Exiting..."; exit; 
  } fi;
# =-=-=-=-=-=-=-=-=-=- Read in the Variables -=-=-=-=-=-=-=-=-=-=-=
  if  [ ! -e $1 ] ; then {
  $ECHO "No viable config file given."; exit 0
  } fi;
# If we're here, there must be a config file to read, the line below
# reads the variables from that file into this bash environment.
  . $1 
# # # Update 2017-01-01 SRJ The line above sources the variables from
# # # the config file into this script - see /Backup/00-System.conf
# # # for more information.
 
# Make sure that the folder list exists, create it if needed;
  if [ ! -f $SNAPSHOT_RW/Backups ] ; then {
  $ECHO Nothing >> $SNAPSHOT_RW/Backups 
  } fi; 
# The full name of the oldest snapshot, first line in Backups
  [ $($CAT $SNAPSHOT_RW/Backups | $WC --lines) -ge $NUM_SNAP ] && \
  OLDEST=$($HEAD -1 $SNAPSHOT_RW/Backups) || OLDEST="Nothing"
# The full name of the last snapshot, last line in Backups
	LAST=$($TAIL -1 $SNAPSHOT_RW/Backups)
# The full name of the snapshot for this itteration. (2006-06-22-09-47)
	CURRENT=$SNAPSHOT_RW/$($DATE +%F-%H-%M)
# What time is it, in seconds past the epoch?
        StartTime=$($DATE +%s)
# Hour, minute, second function;
_hms()
{
     local S=${1}     # this S is used only in this sub-routine
     ((h=S/3600))     # fills h with the integer of seconds / 3600
     ((m=S%3600/60))  # fills m with the integer of the modula from above / 60
     ((s=S%60))       # fills s with the integer of seconds / 60
     printf "%02d:%02d:%02d\n" $h $m $s  # print 2 digit zero padded numbers
}
 
# =-=-=-=-=-=-=-=-=- The script itself. -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
# I stopped mounting and unmounting the filesystem, I now do this by 
# hand on an as-needed basis - these backup scripts are now on the 
# backup drives themselves.
# Attempt to remount the RO mount point as RW; else abort
#  $MOUNT -o remount,rw $MOUNT_DEVICE $MOUNT_DIR ;
#  if (( $? )); then {
#  $ECHO "Could not remount $MOUNT_DIR read-write"; exit; 
#  } fi;
# Creating the snapshot.
# Delete the oldest snapshot, if it exists:
 
  if [ -e ${OLDEST} ] ; then { $RM -rf ${OLDEST} ;} fi ;
 
# Delete the last filelist and create a new one with proper permissions.
 
  if [ -f $FILELIST ] ; then { $RM -f $FILELIST ;} fi ;
  $TOUCH $FILELIST ; $CHMOD 600 $FILELIST
 
# Rsync from the system into the latest snapshot (notice that rsync 
# behaves like cp --remove-destination by default, so the destination is 
# unlinked first.  If it were not so, this would copy over the other 
# snapshot(s) too!
 
  RSARGS=" --archive --verbose --delete --delete-excluded --numeric-ids \
  --modify-window=1 --hard-links --one-file-system --acls --xattrs \
  --exclude-from="$EXCLUDES" --link-dest=$LAST/ "
 
# This is the line that does everything we're doing here...
  $RSYNC $RSARGS $SNAP_DIR $CURRENT >> $FILELIST;
 
# Add today's snapshot to the list, and remove the oldest;
 
  $ECHO $CURRENT >> $SNAPSHOT_RW/Backups
  $GREP -v $OLDEST $SNAPSHOT_RW/Backups >>  $SNAPSHOT_RW/$$.tmp
  $MV -f $SNAPSHOT_RW/$$.tmp $SNAPSHOT_RW/Backups
  EndTime=$($DATE +%s) ; ElapsedTime=$(( $EndTime - $StartTime ))
  TimeWellSpent=$( _hms ${ElapsedTime})
 
# Write out the summary file.
  $HEAD -2 $FILELIST > $SUMMARY; 
  $ECHO $(( $($GREP -ve /$ $FILELIST | $WC --lines )-5)) \
  New files backed up. >> $SUMMARY
  $ECHO $OLDEST was removed and $CURRENT was created >> $SUMMARY
  $ECHO Elapsed time: $TimeWellSpent >> $SUMMARY
  $TAIL -3 $FILELIST >> $SUMMARY
  $MV  $FILELIST  $CURRENT
 
# And thats it for SNAP_DIR. Now remount the RW snapshot mountpoint as RO 
#  $MOUNT -o remount,ro $MOUNT_DEVICE $MOUNT_DIR ;
#  if (( $? )); then { 
#  $ECHO "Could not remount $MOUNT_DIR readonly"; exit; 
#  } fi;

/Backup/01-Root/Excludes, this excludes file lists things we don’t worry about backing up.

/Backup/*
/home/Shared/Backup
*/Cache/*
/dev/*
/flows/*
/initrd*
/home/*/.gvfs
*/lost+found*
/media/*
/mnt/*
/run/*
/tmp/*
/proc/*
/var/virus*
/var/lib/vmware*
/var/run/cups/certs*

/Backup/01-Root/Summary, is just the best bits of the Filelist, below.

sending incremental file list
created directory /Backup/01-Root/2019-08-14-02-02
718 New files backed up.
/Backup/01-Root/2019-08-04-02-02 was removed and /Backup/01-Root/2019-08-14-02-02 was created
Elapsed time: 00:14:56
sent 1,158,536,441 bytes  received 127,738 bytes  2,045,303.05 bytes/sec
total size is 773,557,672,398  speedup is 667.63

/Backups/01-Root/Backups, This is just a list of the backups that exist on the external drive.

/Backup/01-Root/2019-08-05-02-02
/Backup/01-Root/2019-08-06-02-02
/Backup/01-Root/2019-08-07-02-02
/Backup/01-Root/2019-08-08-02-02
/Backup/01-Root/2019-08-09-02-02
/Backup/01-Root/2019-08-10-02-02
/Backup/01-Root/2019-08-11-02-02
/Backup/01-Root/2019-08-12-02-02
/Backup/01-Root/2019-08-13-02-02
/Backup/01-Root/2019-08-14-02-02

/Backup/01-Root/Filelist, this file is moved to /Backup/01-Root/2019-08-14-02-02/Filelist after the rsync has finished.

sending incremental file list
created directory /Backup/01-Root/2019-08-14-02-02
./
etc/.git/
etc/letsencrypt/
opt/mattermost/logs/mattermost.log
root/
root/.PlotLog
root/.SpeedTest.log
...
...
var/spool/postfix/public/pickup
var/spool/postfix/public/qmgr
var/tmp/
var/tmp/sclhoIPgx
var/tmp/systemd-private-23f097ce9d024a4cbcc72c823a582844-mysqld.service-p5VWfi/tmp/
 
sent 1,158,536,441 bytes  received 127,738 bytes  2,045,303.05 bytes/sec
total size is 773,557,672,398  speedup is 667.63

Days between two dates

This little script reports back how many days fall between two arbitrary dates, or one date and today. If you put in a date in the future, it tells how many days as a negative;

# How many days 'till Christmas?
HowManyDays 2017-12-25
-115

#!/bin/bash
# SRJ 2017-08-31
date1=$(date --utc --date "${1}" +%s)
Ex=$?
# The line above puts the exit status of the date command
# into the Ex variable, the lines below exit the shell if
# there was an error reported.
if [ ! ${Ex} = 0 ]
then
exit 1
fi
 
if [ -z "${2}" ]
  then
  date2=$(date --utc +%s)
  else
  date2=$(date --utc --date "${2}" +%s)
  Ex=$?
  # The line above puts the exit status of the date command
  # into the Ex variable, the lines below exit the shell if
  # there was an error reported.
  if [ ! ${Ex} = 0 ]
  then
  exit 1
  fi
fi
 
diffdays=$(( (date2-date1)/(3600*24) ))
 
if [ ${diffdays} = 0 ]
then
echo "This script requires a valid date in the form yyyy-mm-dd or, for"
echo "example, HowManyDays 2015-09-23 will report the number of days"
echo "that have passed between 2015-09-23 and today."
echo 
echo "If this script is given two dates, it returns the number of days"
echo "between those two dates, if the second date is earlier than the"
echo "first, the result will be a negative number."
echo "HowManyDays 2015-09-23 2016-02-27 will return 157."
exit 1
fi
 
echo ${diffdays}

TinyPad

#!/bin/bash
#      SRJ 2016-09-20 Create reasonable passwords
# This is ~/bin/TinyPad
# This script produces 26 rows of 3 columns of 13 characters each.
# It was written to create ISO-ID-1 Cards to carry in the wallet, with 
# a letter before each row as a reminder (I use row G for Google passwords
# for example.
 
Chars="ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890.!@#$%&-_=+abcdefghijklmnopqrstuvwxyz"
Count=${#Chars}   # Length of $Chars
# You can remove specific characters to avoid ambiguity, O or 0 for instance,
# though, if you print this in Courier, the symbols are quite unique.
 
Ran()
{
# This function generates a random number, returns the modulus of that number 
# in the range of 1 to (Number of characters in $Char), and then returns the 
# single character from $Char at that position
Offset=$(($(head -c4 /dev/urandom | od -N2 -tu2 | sed -ne '1s/.* //p')%$Count))
echo "${Chars:${Offset}:1}"
};
 
for Row in A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
do echo -n "${Row} "
  for Column in 1 2 3
  do for Char in 1 2 3 4 5 6 7 8 9 10 11 12 13
    do echo -n "$(Ran)"
    done  ; echo -n " "
  done ; echo ""
done
# The 3 embedded loops above spit out the Rows, Columns, and Char to form an
# array that is designed to fit on a credit card for your wallet. 
 
# For the line below to work properly, you have to install the enscript package,
# then add a media definition in the form;
# Media:	Card		242	153	5	9	228	145 
# to the /etc/enscript.cfg file
# TinyPad | enscript -BM Card --margins=10:9 --output=- -f CourierBold8/9 | ps2pdf - Pad.pdf
# Produces a file, Pad.pdf that can be printed directly with the card printer 
# at the front desk.

Pad.pdf

Perfect_Printers

I wrote this a loong time ago, it amuses me to see that it still seems to work.
The comments in this one make it pretty self-explanatory;

#!/bin/sh
# This gives a list of only the perfectly supported printers under Linux
# It makes a nice list to take with you when you're going to the store.
# 
# The current list of printers include these manufacterers;
# Alps Anitech Apollo Apple Avery Brother CalComp Canon Casio Citizen 
# CItoh Compaq DEC Dell Dym Epson Fuj Fujifilm Fujitsu Generic Genicom
# Gestetner Heidelberg Hitachi HP IBM Imagen Imagistics InfoPrint 
# Infotec Kodak KONIC Kyocera Lanier LaserMaster Lexmark Minolta 
# Mitsubishi NEC NFCP NRG Oce Oki Olivetti Olympus Panasonic PCPI 
# Pentax Pixma Printrex QMS Raven Ricoh Samsung Savin Seiko Sharp 
# SiPix Sony Star Tally TallyGenicom Tektronix Toshiba Tysso Xante 
# Xerox
# 
# Edit the line below to include the printer manufacturers that you have
# an interest in, then run this script.
 
Manufacturers="Brother Epson HP"
 
for Each in $Manufacturers 
 do
  echo "=-=-=-=- $Each -=-=-=-="
  elinks -no-numbering -dump 1 http://www.openprinting.org/printers/manufacturer/$Each \
  | sed -e '1,/Perfect/d' -e '/Mostly/,$d' -e '/^$/d'
 done

Bash_Boggle

I wrote this little script one evening a long time ago, for someone who wanted to take pre-printed sheets to work, as it was more stimulating than the work…

#!/bin/bash
# Print out a Boggle board to stdout.
# The 16 dice and their 6 sides each, this creates Dice[n]=
Dice=\
(QBAJOM EFIEYH ODENWS HPSEIN GKYLUE ILUWRG CITAOA BYLTIA \
 DZNVEA ODUNKT RHASOM PAECMD GVZTEI OFIRXB REALSC PELUTS)
#
AllDice="  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15"
 
# Cut the string that represents the order of the dice in 
# the tray 99 time at some random point, then swap the parts
# around. Repeat as needed.
for ((a=1; a <= 100 ; a++))
do
   LeftCut=$(($(($(($RANDOM%15))*3))+3))
   RightCut=$((LeftCut+3))
   LeftEnd=${AllDice:0:LeftCut} 
   RightEnd=${AllDice:RightCut}
   Middle=${AllDice:LeftCut:3}
   AllDice="$Middle$LeftEnd$RightEnd"
done
# Roll the Dice to get what side is up.
Count=0
for Each in $AllDice
    do
    Side=$(($RANDOM%6))
    Die=${Dice[Each]:Side:1}
    [ $Die == Q ] && echo -n $Die"u " || echo -n $Die"  "
    [ $((Count%4)) -eq 3 ] && echo
    (( Count++ ))
    done

Pad

This generates a list of passwords, as a pad, I carry a subset of one of these with me in case I need a truly random password while I’m out.

#!/bin/bash
#      SRJ 2016-09-20 Create reasonable passwords
#      0        1         2         3         4         5         6         7
#      1234567890123456789012345678901234567890123456789012345678901234567890123456789
Chars="ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890.!@#$%&-_=+abcdefghijklmnopqrstuvwxyz"
# The column labels are taken from the beginning of the $Chars string
Count=${#Chars}   # Length of $Chars
R="60"   # Number of rows in output, 60 works well
C="6"   # Number of columns in output, 6 is the way
L="12"  # Length of string, 12 is ideal
# Shouldn't need to change anything below here
 
if [ ! -z ${1} ] ; then R="${1}" ; fi
if [ ! -z ${2} ] ; then C="${2}" ; fi
if [ ! -z ${3} ] ; then L="${3}" ; fi
 
d=$((L/2)) ; ((d++)) # Half of L plus 1
Sep="-"   # The Seperator at the top of each column
Blanks="" # This line and next generate the empty space at the beginning of each line
for z in $(seq 1 ${#R}) ; do Blanks+=" " ; done
 
Ran()
{
Offset=$(($(head -c4 /dev/urandom | od -N2 -tu2 | sed -ne '1s/.* //p')%$Count))
echo "${Chars:${Offset}:1}"
};
 
# Generate the top two lines of the report, labeling the columns and adding
# a seperator line
for Row in $(seq -w 0 1)
do echo -n "${Blanks}| "
  for Column in $(seq -w 1 ${C})
  do for Place in $(seq -w 1 ${L})
    do if [ ${Row} = "1" ] ; then
      echo -n "${Sep}"
      else if [ ${Place} -eq ${d} ] ; then
        echo -n ${Chars:$((Column-1)):1}
        else echo -n " " ; fi ; fi
    done ; if [ ! ${Column} -eq ${C} ] ; then echo -n " " ; fi
  done ; echo ""
done
 
# Send out the rows and columns of random characters
for Row in $(seq -w 1 ${R})
do echo -n "${Row}| "
  for Column in $(seq -w 1 ${C})
  do for Place in $(seq -w 1 ${L})
    do echo -n "$(Ran)"
    done ; if [ ! ${Column} -eq ${C} ] ; then echo -n " " ; fi
  done ; echo ""
done