scripts/full-system-backup/full-system-backup

#!/bin/bash

#DEBUG="true"
#DEBUG_CLEANUP="true"

CONFIG="$1"
[ -n "$CONFIG" ] || CONFIG="/etc/backup.conf"
[ -f "$CONFIG" ] || exit 1

DATE="$(date +%Y-%m-%d)"

# $DATE must be the second field! (seperator=.)
BACKUP_PREFIX="$(hostname).$DATE"

umask 0077
tmpdir=$(mktemp -d) || exit 1

. $CONFIG

[ -n "$LOCAL_STORAGE_PATH" -a ! -d "$LOCAL_STORAGE_PATH" ] && {
	echo "directory does not exist: <${LOCAL_STORAGE_PATH}>"
	exit 1
}

# defaults to gzip (none, lzop, gzip, bzip2 are allowed)
[ -n "$BACKUP_COMPRESS" ] || BACKUP_COMPRESS="gzip"

# log to stderr when debugging
[ -z "$DEBUG" ] && exec 2>${tmpdir}/msg

filesize() {
	du -shcx $1 2>/dev/null | awk '/total/{ printf("%4s",$1); }'
}

filestat() {
	format="[%A] %U(%u):%G(%g)"
	[ -b "$1" -o -c "$1" ] && format="$format Maj:%T Min:%t"
	stat -c "$format %N" $1 2>/dev/null
}

storage_filename() {
	FILENAME="$1"
	COMPRESS="$2"

	case "$COMPRESS" in
	none)	echo "$FILENAME"
		;;

	lzop)	echo "${FILENAME}.lzo"
		;;

	bzip2)	echo "${FILENAME}.bz2"
		;;

	*)	echo "${FILENAME}.gz"
		;;
	esac
}

storage_put() {
	FILENAME="$1"
	COMPRESS="$2"

	case "$COMPRESS" in
	none)	comp_bin="cat"
		;;

	lzop)	comp_bin="lzop -c"
		;;

	bzip2)	comp_bin="bzip2 -c"
		;;

	*)	comp_bin="gzip -c"
		;;
	esac

	# put file in local storage directory and copy it later to remote storage
	if [ -n "$LOCAL_STORAGE_PATH" ]; then
		$comp_bin > ${LOCAL_STORAGE_PATH}/${FILENAME}
		echo "$FILENAME: ($(filesize ${LOCAL_STORAGE_PATH}/${FILENAME}))" >&2

	# copy file only via ssh
	elif [ "$REMOTE_STORAGE_TYPE" == "ssh" ]; then
		$comp_bin | ssh $REMOTE_STORAGE_HOST "cat > ${REMOTE_STORAGE_PATH}/${FILENAME}"

	# copy file only via ftp
	elif [ "$REMOTE_STORAGE_TYPE" == "ftp" ]; then
		$comp_bin | ncftpput $ftp_cred -U 0077 -V -c $REMOTE_STORAGE_HOST "${REMOTE_STORAGE_PATH}/${FILENAME}"
	fi
}

# assumes '.' as seperator and date (YYYY-MM-DD) in 2nd field
# eg. $(hostname).$(date).tar.gz)
calcold() {
	filelist="$1"
	keep="$2"
	sed 's/.*\([[:digit:]]\{4\}-[[:digit:]]\{2\}-[[:digit:]]\{2\}\).*/\1/' $filelist > ${filelist}.dates
	date -d "-${keep}" "+%Y-%m-%d-XXX" >> ${filelist}.dates
	sort -r -u < ${filelist}.dates > ${filelist}.dates.uniq

	cnt=0
	mark=0
	for datestamp in $(cat $filelist.dates.uniq); do
		if $(echo $datestamp | grep -q "XXX"); then
			mark=1
			continue;
		fi

		cnt=$[cnt +1]
		# keep at least this +1 backups, even if it's older
		if [ "$cnt" -gt 1 -a "$mark" = "1" ]; then
			grep "$datestamp" $filelist
		fi
	done
}

echo "$(basename $0) running on $(hostname) [${DATE}]" >&2

if [ -n "$LOCAL_STORAGE_PATH" ]; then
	echo "LOCAL_STORAGE: $LOCAL_STORAGE_PATH (keep $LOCAL_STORAGE_KEEP)" >&2
else
	echo "LOCAL_STORAGE: none" >&2
fi

case "$REMOTE_STORAGE_TYPE" in
ftp)	# ftp server
	ftp_user=$(echo $REMOTE_STORAGE_CRED | cut -d':' -f1)
	ftp_pass=$(echo $REMOTE_STORAGE_CRED | cut -d':' -f2)
	ftp_cred="-u $ftp_user -p $ftp_pass"

	[ -z "$REMOTE_STORAGE_PATH" ] && REMOTE_STORAGE_PATH="/"
	echo "REMOTE_STORAGE: ftp://${REMOTE_STORAGE_HOST}/${REMOTE_STORAGE_PATH} (keep $REMOTE_STORAGE_KEEP)" >&2
	;;

ssh|scp)
	[ -z "$REMOTE_STORAGE_PATH" ] && REMOTE_STORAGE_PATH="."
	[ -n "$REMOTE_STORAGE_BANDWIDTH" ] && REMOTE_STORAGE_BANDWIDTH="-l $REMOTE_STORAGE_BANDWIDTH"
	echo "REMOTE_STORAGE: ssh://${REMOTE_STORAGE_HOST}:${REMOTE_STORAGE_PATH} (keep $REMOTE_STORAGE_KEEP)" >&2
	;;

*)	echo "REMOTE_STORAGE: none" >&2
	REMOTE_STORAGE_TYPE="none"
	;;
esac

created_files=""
for name in $BACKUPS; do
	eval type=\${BACKUP_${name}}
	eval comp=\${BACKUP_${name}_COMPRESS}

	[ -n "$comp" ] || comp="$BACKUP_COMPRESS"

	echo -e "\nBACKUP_$name: (type: ${type})" >&2

	case "$type" in
	tar)	# tar.gz of files
		eval files=\${BACKUP_${name}_FILES}
		eval exclude=\${BACKUP_${name}_EXCLUDE}

		for i in $files; do
		    echo "  adding $i" >&2
		done

		excluded="--exclude $tmpdir"
		[ -n "$LOCAL_STORAGE_PATH" ] && excluded="$excluded --exclude $LOCAL_STORAGE_PATH"
		for tmp in $exclude; do
			echo "  excluding $(filestat $tmp)" >&2
			excluded="${excluded} --exclude ${tmp}"
		done

		filename="$(storage_filename ${BACKUP_PREFIX}.${name}.tar $comp)"
		[ -n "$files" ] && tar --numeric-owner -C / -cf - $excluded $files | storage_put $filename $comp
		;;

	mysql)	# mysql dump of some/all tables
		eval user=\${BACKUP_${name}_USER}
		eval pass=\${BACKUP_${name}_PASS}
		eval db_list=\${BACKUP_${name}_DBS}

		auth="-u${user} -p${pass}"
		[ -n "$pass" ] || auth="-u ${user}"

		mysql_cmd="/usr/bin/mysql ${auth} -N"
		mysql_schema_dump="/usr/bin/mysqldump ${auth} --opt --all --no-data"
		mysql_data_dump="/usr/bin/mysqldump ${auth} --opt --add-locks=FALSE --no-create-info"

		created_mysql_files=""
		if [ -z "$db_list" ]; then
			db_list=$(echo "SHOW databases;" | $mysql_cmd)

			echo "  privileges" >&2
			sql="SELECT DISTINCT CONCAT(\"'\", user, \"'@'\", host, \"'\") FROM  mysql.user ORDER BY user, host;"
			db_accounts=$(echo $sql | $mysql_cmd)

			for account in $db_accounts; do
				sql="SHOW GRANTS FOR $account;"
				echo "-- $sql"
				echo $sql | $mysql_cmd
				echo

			done >> ${tmpdir}/privileges.sql
			created_mysql_files="privileges.sql"
		fi

		for db in $db_list; do
			echo "  database $db" >&2
			$mysql_schema_dump --databases $db > ${tmpdir}/${db}-schema.sql
			$mysql_data_dump --databases $db > ${tmpdir}/${db}-data.sql
			created_mysql_files="$created_mysql_files ${db}-schema.sql ${db}-data.sql"
		done

		filename="$(storage_filename ${BACKUP_PREFIX}.${name}.tar $comp)"
		[ -n "$created_mysql_files" ] && tar -C $tmpdir -cf - $created_mysql_files | storage_put $filename $comp
		;;

	ldap)	# slapcat of slapd database
		filename="$(storage_filename ${BACKUP_PREFIX}.${name}.ldif $comp)"
		slapcat | storage_put $filename $comp
		;;

	*)	echo "Invalid backup type: <${type}>" >&2
		continue
		;;
	esac
	created_files="$created_files $filename"
done

[ -n "$created_files" ] || {
	echo -e "\nERROR(?): no files created." >&2
	rm -rf $tmpdir
	exit
}

# store a log (only when not debugging)
if [ -z "$DEBUG" ]; then
	cat ${tmpdir}/msg | storage_put ${BACKUP_PREFIX}.log none
	created_files="$created_files ${BACKUP_PREFIX}.log"
fi

if [ -n "$LOCAL_STORAGE_PATH" ]; then

	local_files=""
	# check files & prefix with local path
	for created in $created_files; do
		[ -e "${LOCAL_STORAGE_PATH}/${created}" ] && local_files="$local_files ${LOCAL_STORAGE_PATH}/${created}"
	done

	# copy local files to remote storage
	case "$REMOTE_STORAGE_TYPE" in
	ftp)	echo -e "\ncopy archives to REMOTE_STORAGE" >&2
		ncftpput $ftp_cred -U 0077 -V $REMOTE_STORAGE_HOST $REMOTE_STORAGE_PATH $local_files
		;;

	ssh)	echo -e "\ncopy archives to REMOTE_STORAGE" >&2
		scp -p -q $REMOTE_STORAGE_BANDWIDTH $local_files "${REMOTE_STORAGE_HOST}:${REMOTE_STORAGE_PATH}"
		;;
	esac

	# cleanup local storage
	echo -e "\nremoving old (> $LOCAL_STORAGE_KEEP) backups from LOCAL_STORAGE" >&2

	{ cd ${LOCAL_STORAGE_PATH}; ls -1 $(hostname).* > ${tmpdir}/local.files; }
	local_old_files=$(calcold ${tmpdir}/local.files "$LOCAL_STORAGE_KEEP")

	for old_file in $local_old_files; do
		echo "  removing $old_file ($(filesize $old_file))" >&2
		[ -z "$DEBUG_CLEANUP" ] && rm -f ${LOCAL_STORAGE_PATH}/${old_file}
	done
fi

# cleanup remote storage
case "$REMOTE_STORAGE_TYPE" in
ftp)	echo -e "\nremoving old (> $REMOTE_STORAGE_KEEP) backups from REMOTE_STORAGE" >&2

	ncftpls $ftp_cred "ftp://${REMOTE_STORAGE_HOST}${REMOTE_STORAGE_PATH}" > ${tmpdir}/remote.files
	remote_old_files=$(calcold ${tmpdir}/remote.files "$REMOTE_STORAGE_KEEP")

	for old_file in $remote_old_files; do
		echo "  removing $old_file" >&2
		[ -z "DEBUG_CLEANUP" ] && \
			ncftpls $ftp_cred -X "DELE $old_file" "ftp://${REMOTE_STORAGE_HOST}${REMOTE_STORAGE_PATH}" > /dev/null
	done
	;;

ssh|scp)
	echo -e "\nremoving old (> $REMOTE_STORAGE_KEEP) backups from REMOTE_STORAGE" >&2

	ssh $REMOTE_STORAGE_HOST "cd ${REMOTE_STORAGE_PATH}; ls -1 $(hostname).*" > ${tmpdir}/remote.files
	remote_old_files=$(calcold ${tmpdir}/remote.files "$REMOTE_STORAGE_KEEP")

	if [ -n "$remote_old_files" ]; then
		remote_old_paths=""
		for old_file in $remote_old_files; do
			echo "  removing $old_file" >&2
			remote_old_paths="$remote_old_paths ${REMOTE_STORAGE_PATH}/${old_file}"
		done
		[ -z "$DEBUG_CLEANUP" ] && ssh $REMOTE_STORAGE_HOST "rm $remote_old_paths"
	fi
	;;
esac

[ -n "$MAILADDRESS" -a -z "$DEBUG" ] && mail -s "full-system-backup on $(hostname)" $MAILADDRESS < ${tmpdir}/msg

rm -rf $tmpdir