Backup der VMs

Die VM´s werden via zwei Backup-scripte (auf dem KVM) automatisch mit einem Crontab monatlich inkrementell auf die Storeage Box gesichert:

crontab -e
20 1 1 */2 * /storage/vms/backup-part1.sh 2>&1 | msmtp michael@goedeke.de
20 1 2 */2 * /storage/vms/backup-part2.sh 2>&1 | msmtp michael@goedeke.de
20 1 3 */2 * /storage/vms/backup-part3.sh 2>&1 | msmtp michael@goedeke.de

/storage/vms/backup-vms.sh

#!/bin/bash

#################### config variables########################

#Uncomment to backup domains myVMa and myVMb
DOMAINS="<alle in part1 bzw. in part2>"

#Alternatively list all VMs and back them all up
#DOMAINS=$(virsh list --all | tail -n +3 | awk '{print $2}')

BACKUPROOT=/var/backups/kvms/backup_vms/$(hostname)

LOG="$BACKUPROOT/logs/qemu-backup.$(date +%Y-%m-%d).log"
SNAPPREFIX=snaptemp-

# Path to XML files for stopped VM...
PATH_LIBVIRT_QEMU="/etc/libvirt/qemu"

#pause before blockcommit
pause_domain_bc=false
#pause after blockcommit fail. if pause_domain_bc is set to true, this option is skiped
pause_domain_after_bc_fail=true
#on fail blockcommit try blockjob
enable_bj_attemts=true
blockjob_retrycount=1
blockjob_delay="30s"
pause_while_bj_attemtps=true
#if blockjob attempts fail, try pause domain? (if pause_while_bj_attemtps is set to true, script ignore this option)
pause_domain_after_bj_fail=true

EMR="michael@goedeke.de"
#****************************
#you can create conf file with variables which overwrite those above
#fo example if name of yours script is backup_vms.sh,create file in location wher is your script and name him->  backup_vms.conf
#if you want other name and path, you need to find and change value for variable below (outsite config variables block) named-> confFile
#it is usefull if we work with many hypervisiors and make some changes in script
#****************************

#################### end of config variables#################

SHCMD="$(basename -- $0)"
SHCMD_path=$(dirname $(readlink -f $0))
confFile="$SHCMD_path/${SHCMD%.*}.conf"
if [ -f "$confFile" ]; then
    source "$confFile"
    echo "overwrite vars from config file"
else
    echo "no config file, but if you set variables on begining it is not necessary"
fi

[ ! -f $BACKUPROOT/logs ] && mkdir -p $BACKUPROOT/logs
DATE="$(date +%Y-%m-%d.%H%M%S)"

ERRORED=0
COMMITSNAPTAB[0]=false
BREAK=false






# extract the date coding in FILEFULLNAME (note: filename format must be YYYY-MM-DD)
dtmatch () { sed -n -e 's/.*\(2[0-1][0-9][0-9]-[0-1][0-9]-[0-3][0-9]\).*/\1/p'; }
#check if previous instance of script is not running, it's mandatory for blockcommit after interput script from some reason
if pidof -x "$SHCMD" -o $$ >/dev/null;then
	ERR="An another instance of this script is already running, please clear all the sessions of this script before starting a new one
	If you are sure to stop previous instance, try command: kill -9 \$(pidof -x \"$SHCMD\")"
	echo "$ERR" >> $LOG
	echo "$ERR"
	exit 1
fi
echo "$SHCMD: Starting backups on $(date +'%d-%m-%Y %H:%M:%S')"  >> $LOG
for DOMAIN in $DOMAINS; do
	BREAK=false
	#check if domain exists
	VMSRC=$(virsh list --all | grep [[:space:]]$DOMAIN[[:space:]] | awk '{print $3}')
	VMSRC=${#VMSRC}
	if [[ $VMSRC -eq 0 ]]; then
		ERR="Domain $DOMAIN on this hypervisior not exists"
		echo "$ERR" >> $LOG
		echo "$ERR"	
		continue
	fi
	
	echo "---- VM Backup start $DOMAIN ---- $(date +'%d-%m-%Y %H:%M:%S')"  >> $LOG

	VM_RUNNING=1
	
	VMSTATE=$(LC_ALL=en_EN virsh list --all | grep [[:space:]]$DOMAIN[[:space:]] | awk '{print $3}')
	echo "$VMSTATE"
	if [[ $VMSTATE == "running" || $VMSTATE == "paused" ]]; then
		MSG="-> VM $DOMAIN running or paused."
		echo "$MSG" >> $LOG
		echo "$MSG"		
	else

		MSG="-> VM $DOMAIN not running. No snapshot and blockcommit. Only copy."
		echo "$MSG" >> $LOG
		echo "$MSG"
    	VM_RUNNING=0
	fi

	BACKUPFOLDER=$BACKUPROOT/$DOMAIN
	[ ! -d $BACKUPFOLDER ] && mkdir -p $BACKUPFOLDER
	TARGETS=$(virsh domblklist $DOMAIN --details | grep disk | awk '{print $3}')
	IMAGES=$(virsh domblklist $DOMAIN --details | grep disk | awk '{print $4}')

	# check to make sure the VM is running on a standard image, not
	# a snapshot that may be from a backup that previously failed
	unset COMMITSNAPTAB[*]
	i=0
	COMMITSNAP=false
	for IMAGE in $IMAGES; do
		set -o noglob
		if [[ $IMAGE == *${SNAPPREFIX}* ]]; then
			set +o noglob
			if [[ $VM_RUNNING -eq 0 ]]; then
				ERR="$SHCMD: Error VM $DOMAIN is not running but is on a snapshot disk image: $IMAGE"
				echo $ERR >> $LOG
				echo "$ERR"
				echo "$ERR
Host:       $HOST
Disk Image: $IMAGE
Domain:     $DOMAIN
Command:    virsh domblklist $DOMAIN --details" | mail -s "$SHCMD snapshot Exception for $DOMAIN" $EMR
				BREAK=true
				ERRORED=$(($ERRORED+1))
				break
			else
				COMMITSNAPTAB[$i]=true
				COMMITSNAP=true
				MSG="VM $DOMAIN is running but is on snapshot image $IMAGE so mark him for blockcommit first"
				echo "$MSG" >> $LOG
				echo "$MSG"
			fi
		else
			# if not running on snapshot still must check if ther is no tmp file, if is presents, must be deleted before start new snapshot
			FILEPATH="${IMAGE%/*}"
			FILEFULLNAME=`basename "$IMAGE"`
			FILENAME="${FILEFULLNAME%.*}"
			#FILEEXT="${FILEFULLNAME#*.}
			TMPFILE="$FILEPATH/$FILENAME.$SNAPPREFIX$DOMAIN"
			if [ -f "$TMPFILE" ]; then
				CMD="rm -f $TMPFILE >> $LOG 2>&1"
				MSG=" Deleting temporary image $TMPFILE after making snapshot"
				echo "$MSG" >>$LOG
				echo "$MSG"
				echo "Command: $CMD" >> $LOG
				echo "Command: $CMD"
				eval "$CMD"
			fi
		fi
		set +o noglob
		i=$i+1
	done
	[ $BREAK == true ] && continue

	if [[ $VM_RUNNING -eq 1 ]]; then
		#if vm running on snap, first i need to merge
		#i=0
		if [[ $COMMITSNAP == true ]]; then
			i=0
			for TARGET in $TARGETS; do
				if [[ ${COMMITSNAPTAB[$i]} == true ]]; then
					if [[ $VMSTATE == "paused" ]]; then
						pause_enabled=true
					else
						pause_enabled=false
					fi
					if [[ $pause_domain_bc == true && $pause_enabled == false ]]; then
						MSG="Try to pause domain $DOMAIN first"
						echo "$MSG" >> $LOG
						echo "$MSG"
						CMDst="virsh suspend $DOMAIN"
						echo "Command: $CMDst" >> $LOG
						echo "Command: $CMDst"
						eval "$CMDst"
						if [ $? -ne 0 ]; then
							ERR="can't pause domain $DOMAIN, continue anyway"
							echo "$ERR" >> $LOG
							echo "$ERR"
						else
							pause_enabled=true
						fi
					fi
					MSG="$TARGET BLOCKCOMMIT previous snapshot "
					echo "$MSG" >> $LOG
					echo "$MSG"
					CMD="virsh blockcommit $DOMAIN $TARGET --active --pivot >> $LOG 2>&1"
					echo "Command: $CMD" >> $LOG
					echo "Command: $CMD"
					eval "$CMD"
					if [ $? -ne 0 ]; then
						ERR="Could not merge changes for disk $TARGET of $DOMAIN with blockcommit. VM may be in an invalid state."
						echo "$ERR" >> $LOG
						echo "$ERR"
						echo "$ERR
Host:    $HOST
Domain:  $DOMAIN
Command: $CMD" | mail -s "$SHCMD blockcommit Exception for $DOMAIN" $EMR
						if [[ $pause_domain_after_bc_fail == true && $pause_enabled == false ]]; then
							MSG="Try to pause domain $DOMAIN and retry bc"
							echo "$MSG" >> $LOG
							echo "$MSG"
							CMDst="virsh suspend $DOMAIN"
							echo "Command: $CMDst" >> $LOG
							echo "Command: $CMDst"
							eval "$CMDst"
							if [ $? -ne 0 ]; then
								ERR="can't pause domain $DOMAIN, continue anyway"
								echo "$ERR" >> $LOG
								echo "$ERR"
							else
								pause_enabled=true
							fi
							MSG="$TARGET BLOCKCOMMIT previous snapshot "
							echo "$MSG" >> $LOG
							echo "$MSG"
							CMD="virsh blockcommit $DOMAIN $TARGET --active --pivot >> $LOG 2>&1"
							echo "Command: $CMD" >> $LOG
							echo "Command: $CMD"
							eval "$CMD"
							if [ $? -ne 0 ]; then
								ERR="Could not merge changes for disk $TARGET of $DOMAIN (paused) with blockcommit. VM may be in an invalid state."
								echo "$ERR" >> $LOG
								echo "$ERR"
								echo "$ERR
Host:    $HOST
Domain:  $DOMAIN
Command: $CMD" | mail -s "$SHCMD blockcommit Exception for $DOMAIN" $EMR
							else
								#if bc succes need to disable bj, so set to false
								enable_bj_attemts=false
							fi
						fi	
						if [ $enable_bj_attemts == true ]; then
							if [[ $pause_while_bj_attemtps == true ]]; then
								MSG="Try to pause domain $DOMAIN first"
								echo "$MSG" >> $LOG
								echo "$MSG"
								CMDst="virsh suspend $DOMAIN"
								echo "Command: $CMDst" >> $LOG
								echo "Command: $CMDst"
								eval "$CMDst"
								if [ $? -ne 0 ]; then
									ERR="can't pause domain $DOMAIN, continue anyway"
									echo "$ERR" >> $LOG
									echo "$ERR"
								else
									pause_enabled=true
								fi
							else
								if [[ $pause_enabled == true && $VMSTATE == "running" ]]; then
									MSG="Resume domain $DOMAIN"
									echo "$MSG" >> $LOG
									echo "$MSG"
									CMDst="virsh resume $DOMAIN"
									echo "Command: $CMDst" >> $LOG
									echo "Command: $CMDst"
									eval "$CMDst"
									if [ $? -ne 0 ]; then
										ERR="can't resume domain $DOMAIN, continue anyway"
										echo "$ERR" >> $LOG
										echo "$ERR"
									else
										pause_enabled=false
									fi
								fi
							fi
						
							for (( j=1; j<=$blockjob_retrycount; j++ )) do
								CMD="virsh blockjob --domain $DOMAIN --pivot $TARGET"
								echo "Command: $CMD" >> $LOG
								echo "Command: $CMD"
								eval "$CMD"
								if [ $? -ne 0 ]; then
									ERR="virsh blockjob attempt no $j --domain $DOMAIN --pivot $TARGET failed"
									echo "$ERR" >> $LOG
									echo "$ERR"
									if [[ $j -eq $blockjob_retrycount ]]; then
										ERR="all blockjob attempts --domain $DOMAIN --pivot $TARGET failed."
										echo "$ERR" >> $LOG
										echo "$ERR"
										echo "$ERR
Host:    $HOST
Domain:  $DOMAIN
Command: $CMD" | mail -s "$SHCMD blockjob  Exception for $DOMAIN" $EMR
										if [[ $pause_domain_after_bj_fail == true && $pause_enabled == false && $VMSTATE == "running" ]]; then
											MSG="Try to pause domain $DOMAIN first"
											echo "$MSG" >> $LOG
											echo "$MSG"
											CMDst="virsh suspend $DOMAIN"
											echo "Command: $CMDst" >> $LOG
											echo "Command: $CMDst"
											eval "$CMDst"
											if [ $? -ne 0 ]; then
												ERR="can't pause domain $DOMAIN"
												echo "$ERR" >> $LOG
												echo "$ERR"
											else
												echo "Command: $CMD" >> $LOG
												echo "Command: $CMD"
												eval "$CMD"
													if [ $? -ne 0 ]; then
														ERR="can't blockjob --piovot $TARGET anyway, I'm giving up"
														echo "$ERR" >> $LOG
														echo "$ERR"
														echo "$ERR
Host:    $HOST
Domain:  $DOMAIN
Command: $CMD" | mail -s "$SHCMD blockjob Exception for $DOMAIN" $EMR
														echo "$ERR" >> $LOG
														echo "$ERR"
														BREAK=true
														ERRORORED=$(($ERRORED+1))
													fi
												MSG="Resume domain $DOMAIN"
												echo "$MSG" >> $LOG
												echo "$MSG"
												CMDst="virsh resume $DOMAIN"
												echo "Command: $CMDst" >> $LOG
												echo "Command: $CMDst"
												eval "$CMDst"
												if [ $? -ne 0 ]; then
													ERR="can't resume domain $DOMAIN, continue anyway"
													echo "$ERR" >> $LOG
													echo "$ERR"
												else
													pause_enabled=false
												fi
											fi
										else
											BREAK=true
											ERRORORED=$(($ERRORED+1))
											break
										fi
									fi
									sleep $blockjob_delay
								else
									MSG="virsh blockjob attempt no $j --domain $DOMAIN --pivot $TARGET success"
									echo "$MSG" >> $LOG
									echo "$MSG"
									break
								fi
							done
							[ $BREAK == true ] && break
						fi
					fi
					if [[ $pause_enabled == true && $VMSTATE == "running" ]]; then
						MSG="Resume domain $DOMAIN"
						echo "$MSG" >> $LOG
						echo "$MSG"
						CMDst="virsh resume $DOMAIN"
						echo "Command: $CMDst" >> $LOG
						echo "Command: $CMDst"
						eval "$CMDst"
						if [ $? -ne 0 ]; then
							ERR="can't resume domain $DOMAIN, continue anyway"
							echo "$ERR" >> $LOG
							echo "$ERR"
						else
							pause_enabled=false
						fi
					fi
				fi
				i=$i+1
			done
			[ $BREAK == true ] && continue
			#delete tmp images after blockcommit
			for IMAGE in $IMAGES; do
				set -o noglob
				if [[ $IMAGE == *${SNAPPREFIX}* ]]; then
					set +o noglob
					CMD="rm -f $IMAGE >> $LOG 2>&1"
					MSG=" Deleting temporary image $IMAGE"
					echo "$MSG" >> $LOG
					echo "$MSG"
					echo "Command: $CMD" >> $LOG
					echo "Command: $CMD"
					eval "$CMD"
				fi
				set +o noglob
			done
			#Reload images after blockcommit
			IMAGES=$(virsh domblklist $DOMAIN --details | grep disk | awk '{print $4}')
		fi
		DISKSPEC=""
		for TARGET in $TARGETS; do
			DISKSPEC="$DISKSPEC --diskspec $TARGET,snapshot=external"
		done

		# transfer the VM to snapshot disk image(s)
			CMD="virsh snapshot-create-as --domain $DOMAIN --name ${SNAPPREFIX}$DOMAIN --no-metadata --atomic --disk-only --quiesce $DISKSPEC >> $LOG 2>&1"
			echo "Command: $CMD" >> $LOG 2>&1
			echo "Command: $CMD"
			eval "$CMD"
			if [ $? -ne 0 ]; then
				MSG="Error create snapshot with --quiesce mode (it's require qemu-guest-agent), try without"
				echo "$MSG" >> $LOG
				echo "$MSG"
				CMD="virsh snapshot-create-as --domain $DOMAIN --name ${SNAPPREFIX}$DOMAIN --no-metadata --atomic --disk-only $DISKSPEC >> $LOG 2>&1"
				echo "Command: $CMD" >> $LOG 2>&1
				echo "Command: $CMD"
				eval "$CMD"
			fi
			if [ $? -ne 0 ]; then
				ERR="Failed to create snapshot for $DOMAIN"
				echo "$ERR" >> $LOG
				echo "$ERR"
				echo "$ERR
Host:    $HOST
Domain:  $DOMAIN
Command: $CMD" | mail -s "$SHCMD snapshot Exception for $DOMAIN" $EMR
				ERRORED=$(($ERRORED+1))
				continue
			fi
		#i=$i+1

	fi

	
	####################################
	#Rsync							   #
	####################################
	MSG="Rsync"
	echo "$MSG" >> $LOG
	echo "$MSG"
	for IMAGE in $IMAGES; do
		FILEFULLNAME=`basename "$IMAGE"`
		if test -f "$BACKUPFOLDER/$FILEFULLNAME"; then
			MSG="Backup exists, merging only changes to image $BACKUPFOLDER/$FILEFULLNAME"
			echo "$MSG" >> $LOG
			echo "$MSG"
			#CMD="rsync -apvhz --inplace --progress $IMAGE $BACKUPFOLDER/$FILEFULLNAME "
			CMD="rsync -apvhz --inplace --progress $IMAGE $BACKUPFOLDER/$FILEFULLNAME "
		else
			MSG="Backup does not exist, creating a full sparse copy of image $IMAGE"
			echo "$MSG" >> $LOG
			echo "$MSG"
			#CMD="rsync -apvhz --sparse --progress $IMAGE $BACKUPFOLDER/$FILEFULLNAME "
			CMD="rsync -apvhz --sparse --progress $IMAGE $BACKUPFOLDER/$FILEFULLNAME "
		fi
		echo "Command: $CMD" >> $LOG
		echo "Command: $CMD"
		eval "$CMD"
	done
	########################################
	# After sync file merge snapshots back #
	########################################

	if [[ $VM_RUNNING -eq 1 ]]; then
		# Update the VM's disk image(s) with any changes recorded in the snapshot
		# while the copy process was running.  In qemu lingo this is called a "pivot"
			BACKUPIMAGES=$(virsh domblklist $DOMAIN --details | grep disk | awk '{print $4}')
			for TARGET in $TARGETS; do
				#if [[ ${COMMITSNAPTAB[$i]} == true ]]; then
					#rsync can tak long time and state could changed so check once again 
					#if state shut there will simple error becouse blockcommi and blockjob can't be done on shutdown domain.
					VMSTATE=$(LC_ALL=en_EN virsh list --all | grep [[:space:]]$DOMAIN[[:space:]] | awk '{print $3}')
					if [[ $VMSTATE == "paused" ]]; then
						pause_enabled=true
					else
						pause_enabled=false
					fi
					if [[ $pause_domain_bc == true && $pause_enabled == false ]]; then
						MSG="Try to pause domain $DOMAIN first"
						echo "$MSG" >> $LOG
						echo "$MSG"
						CMDst="virsh suspend $DOMAIN"
						echo "Command: $CMDst" >> $LOG
						echo "Command: $CMDst"
						eval "$CMDst"
						if [ $? -ne 0 ]; then
							ERR="can't pause domain $DOMAIN, continue anyway"
							echo "$ERR" >> $LOG
							echo "$ERR"
						else
							pause_enabled=true
						fi
					fi
					MSG="$TARGET BLOCKCOMMIT previous snapshot "
					echo "$MSG" >> $LOG
					echo "$MSG"
					CMD="virsh blockcommit $DOMAIN $TARGET --active --pivot >> $LOG 2>&1"
					echo "Command: $CMD" >> $LOG
					echo "Command: $CMD"
					eval "$CMD"
					if [ $? -ne 0 ]; then
						ERR="Could not merge changes for disk $TARGET of $DOMAIN with blockcommit. VM may be in an invalid state."
						echo "$ERR" >> $LOG
						echo "$ERR"
						echo "$ERR
Host:    $HOST
Domain:  $DOMAIN
Command: $CMD" | mail -s "$SHCMD blockcommit Exception for $DOMAIN" $EMR
						if [[ $pause_domain_after_bc_fail == true && $pause_enabled == false ]]; then
							MSG="Try to pause domain $DOMAIN and retry bc"
							echo "$MSG" >> $LOG
							echo "$MSG"
							CMDst="virsh suspend $DOMAIN"
							echo "Command: $CMDst" >> $LOG
							echo "Command: $CMDst"
							eval "$CMDst"
							if [ $? -ne 0 ]; then
								ERR="can't pause domain $DOMAIN, continue anyway"
								echo "$ERR" >> $LOG
								echo "$ERR"
							else
								pause_enabled=true
							fi
							MSG="$TARGET BLOCKCOMMIT previous snapshot "
							echo "$MSG" >> $LOG
							echo "$MSG"
							CMD="virsh blockcommit $DOMAIN $TARGET --active --pivot >> $LOG 2>&1"
							echo "Command: $CMD" >> $LOG
							echo "Command: $CMD"
							eval "$CMD"
							if [ $? -ne 0 ]; then
								ERR="Could not merge changes for disk $TARGET of $DOMAIN (paused) with blockcommit. VM may be in an invalid state."
								echo "$ERR" >> $LOG
								echo "$ERR"
								echo "$ERR
Host:    $HOST
Domain:  $DOMAIN
Command: $CMD" | mail -s "$SHCMD blockcommit Exception for $DOMAIN" $EMR
							else
								#if bc succes need to disable bj, so set to false
								enable_bj_attemts=false
							fi
						fi	
						if [ $enable_bj_attemts == true ]; then
							if [[ $pause_while_bj_attemtps == true ]]; then
								MSG="Try to pause domain $DOMAIN first"
								echo "$MSG" >> $LOG
								echo "$MSG"
								CMDst="virsh suspend $DOMAIN"
								echo "Command: $CMDst" >> $LOG
								echo "Command: $CMDst"
								eval "$CMDst"
								if [ $? -ne 0 ]; then
									ERR="can't pause domain $DOMAIN, continue anyway"
									echo "$ERR" >> $LOG
									echo "$ERR"
								else
									pause_enabled=true
								fi
							else
								if [[ $pause_enabled == true && $VMSTATE == "running" ]]; then
									MSG="Resume domain $DOMAIN"
									echo "$MSG" >> $LOG
									echo "$MSG"
									CMDst="virsh resume $DOMAIN"
									echo "Command: $CMDst" >> $LOG
									echo "Command: $CMDst"
									eval "$CMDst"
									if [ $? -ne 0 ]; then
										ERR="can't resume domain $DOMAIN, continue anyway"
										echo "$ERR" >> $LOG
										echo "$ERR"
									else
										pause_enabled=false
									fi
								fi
							fi
						
							for (( j=1; j<=$blockjob_retrycount; j++ )) do
								CMD="virsh blockjob --domain $DOMAIN --pivot $TARGET"
								echo "Command: $CMD" >> $LOG
								echo "Command: $CMD"
								eval "$CMD"
								if [ $? -ne 0 ]; then
									ERR="virsh blockjob attempt no $j --domain $DOMAIN --pivot $TARGET failed"
									echo "$ERR" >> $LOG
									echo "$ERR"
									if [[ $j -eq $blockjob_retrycount ]]; then
										ERR="all blockjob attempt --domain $DOMAIN --pivot $TARGET failed."
										echo "$ERR" >> $LOG
										echo "$ERR"
										echo "$ERR
Host:    $HOST
Domain:  $DOMAIN
Command: $CMD" | mail -s "$SHCMD blockjob  Exception for $DOMAIN" $EMR
										if [[ $pause_domain_after_bj_fail == true && $pause_enabled == false && $VMSTATE == "running" ]]; then
											MSG="Try to pause domain $DOMAIN first"
											echo "$MSG" >> $LOG
											echo "$MSG"
											CMDst="virsh suspend $DOMAIN"
											echo "Command: $CMDst" >> $LOG
											echo "Command: $CMDst"
											eval "$CMDst"
											if [ $? -ne 0 ]; then
												ERR="can't pause domain $DOMAIN"
												echo "$ERR" >> $LOG
												echo "$ERR"
											else
												echo "Command: $CMD" >> $LOG
												echo "Command: $CMD"
												eval "$CMD"
													if [ $? -ne 0 ]; then
														ERR="can't blockjob --piovot $TARGET anyway, I'm giving up"
														echo "$ERR" >> $LOG
														echo "$ERR"
														echo "$ERR
Host:    $HOST
Domain:  $DOMAIN
Command: $CMD" | mail -s "$SHCMD blockjob Exception for $DOMAIN" $EMR
														echo "$ERR" >> $LOG
														echo "$ERR"
														BREAK=true
														ERRORORED=$(($ERRORED+1))
													fi
												MSG="Resume domain $DOMAIN"
												echo "$MSG" >> $LOG
												echo "$MSG"
												CMDst="virsh resume $DOMAIN"
												echo "Command: $CMDst" >> $LOG
												echo "Command: $CMDst"
												eval "$CMDst"
												if [ $? -ne 0 ]; then
													ERR="can't resume domain $DOMAIN, continue anyway"
													echo "$ERR" >> $LOG
													echo "$ERR"
												else
													pause_enabled=false
												fi
											fi
										else
											BREAK=true
											ERRORORED=$(($ERRORED+1))
											break
										fi
									fi
									sleep $blockjob_delay
								else
									MSG="virsh blockjob attempt no $j --domain $DOMAIN --pivot $TARGET success"
									echo "$MSG" >> $LOG
									echo "$MSG"
									break
								fi
							done
							[ $BREAK == true ] && break
						fi
					fi
					if [[ $pause_enabled == true && $VMSTATE == "running" ]]; then
						MSG="Resume domain $DOMAIN"
						echo "$MSG" >> $LOG
						echo "$MSG"
						CMDst="virsh resume $DOMAIN"
						echo "Command: $CMDst" >> $LOG
						echo "Command: $CMDst"
						eval "$CMDst"
						if [ $? -ne 0 ]; then
							ERR="can't resume domain $DOMAIN, continue anyway"
							echo "$ERR" >> $LOG
							echo "$ERR"
						else
							pause_enabled=false
						fi
					fi
				#fi
				i=$i+1
			done
		[ $BREAK == true ] && continue

		# Now that the VM's disk image(s) have been successfully committed/pivoted to
		# back to the main disk image, remove the temporary snapshot image file(s)
			for BACKUP in $BACKUPIMAGES; do
				set -o noglob
				if [[ $BACKUP == *${SNAPPREFIX}* ]]; then
					set +o noglob
					CMD="rm -f $BACKUP >> $LOG 2>&1"
					MSG=" Deleting temporary image $BACKUP"
					echo "$MSG" >> $LOG
					echo "$MSG"
					MSG="Command: $CMD"
					echo "$MSG" >> $LOG
					echo "$MSG"
					eval "$CMD"
				fi
				set +o noglob
			done

		# capture the VM's definition in use at the time the backup was done
			CMD="virsh dumpxml $DOMAIN > $BACKUPFOLDER/$DOMAIN.xml 2>> $LOG"
			MSG="Command: $CMD"
			echo "$MSG" >> $LOG
			echo "$MSG"
			eval "$CMD"
	else
		# copy the VM's definition
		CMD="cp $PATH_LIBVIRT_QEMU/$DOMAIN.xml $BACKUPFOLDER/$DOMAIN.xml 2>> $LOG"
		MSG="Command: $CMD"
		echo "$MSG" >> $LOG
		echo "$MSG"
		eval "$CMD"
	fi


	MSG="---- Backup done $DOMAIN ---- $(date +'%d-%m-%Y %H:%M:%S') ----"
	echo "$MSG" >> $LOG
	echo "$MSG"
done
MSG="$SHCMD: Finished backups at $(date +'%d-%m-%Y %H:%M:%S')
====================" >> $LOG
echo "$MSG" >> $LOG
echo "$MSG"

exit $ERRORED