@olivierlambert
With dedup i mean dedup of the backup repo. And its an extremely big deal for backup repos. Backup vendors put a lot of effort in this. CommVault and NetBackup have it natively and Veeam does recommend deduplicated storage (Data Domain,...).
In our case we use an S3 backend and if you pay for a month worth of full backups 2-3 times a full backup size or 30 times a compressed full backup size is really a big deal. And we keep out backups sometimes for month.
We are doing bi-weekly fulls with our homebuilt kopia strategy of a 1.3 TB SR (no empty disks) and are using apx. 1.37 TB S3.
its from 0.64 TB up to 1.37 TB. Green is size used and black are the blocks which could be deleted.
if someone is interested why we use kopia instead of restic. restic is super slow on dumps to stdout and on mount. we use mount in combination with libguestfs to do file-level-recovery.
this is the script. not super professional and not the best error-handling but it may be useful:
#!/bin/bash
VMNAME=$1
SERVER=$2
SERVER=${SERVER:=172.25.10.2}
BACKUPDESC="Backup $(date +'%Y-%m-%d %H:%M:%S') of VM $VMNAME"
echo "$(date +'%Y-%m-%d %H:%M:%S') Starting Backup of vm $VMNAME"
echo SERVER:$SERVER
PASS=$(cat /root/.pass)
export XE_EXTRA_ARGS="server=$SERVER,port=443,username=root,password=$PASS"
VMID=$(xe vm-list name-label=$VMNAME | grep uuid | sed "s/.*: //g")
if [ -z "$VMID" ]
then
echo VMID not found
exit 1
fi
echo VMID: $VMID
SNAPID=$(xe vm-snapshot uuid=$VMID new-name-label=backup_$VMNAME)
if [ -z "$SNAPID" ]
then
echo SNAPID not found
exit 1
fi
echo SNAPID: $SNAPID
declare -a DISKID
declare -a DISKNAME
TEMPD=$(mktemp -d)
mkdir $TEMPD/data
echo "$(date +'%Y-%m-%d %H:%M:%S') Temp directory: $TEMPD"
xe vm-export metadata=true uuid=$VMID filename=$TEMPD/data/metadata.ova.tar
xe snapshot-disk-list uuid=$SNAPID vdi-params=all vbd-params=all > $TEMPD/data/vdi.txt
xe vm-list name-label=$VMNAME params=all > $TEMPD/data/vm.txt
ls -l $TEMPD/data
DISKID=( $(xe snapshot-disk-list uuid=$SNAPID vdi-params=uuid,name-label vbd-params=false | grep uuid | awk '{print $5}') )
DISKNAME=( $(xe snapshot-disk-list uuid=$SNAPID vdi-params=uuid,name-label vbd-params=false | grep name-label | awk '{print $4}') )
echo DISKID: ${DISKID[@]}
echo DISKNAME: ${DISKNAME[@]}
for i in ${!DISKID[@]}; do
DN=${DISKNAME[$i]}
DID=${DISKID[$i]}
echo "$(date +'%Y-%m-%d %H:%M:%S') tagging disk $i is ${DID} ${DN}"
xe vdi-param-set name-description="$BACKUPDESC" uuid=$DID
done
for i in ${!DISKID[@]}; do
DN=${DISKNAME[$i]}
DID=${DISKID[$i]}
echo "$(date +'%Y-%m-%d %H:%M:%S') backup disk $i is ${DID} ${DN}"
xe vdi-export uuid=$DID filename= format=raw | kopia snapshot create --tags=vm:$VMNAME,data=$DN --no-progress --log-level=warning --override-source=/vms/$VMNAME/$DN --stdin-file=${DID}.vhd - || echo "ERROR"
sleep 5
xe vdi-destroy uuid=$DID
done
tar cvpf $TEMPD/metadata.tar -C $TEMPD/data .
kopia snapshot create --tags=vm:$VMNAME,data=meta --no-progress --log-level=info --override-source=/vms/$VMNAME $TEMPD/metadata.tar || echo "ERROR"
rm -rf $TEMPD
echo "$(date +'%Y-%m-%d %H:%M:%S') waiting 30s"
sleep 30
echo "$(date +'%Y-%m-%d %H:%M:%S') deleting snapshot $SNAPID of VM $VMNAME"
xe snapshot-uninstall uuid=$SNAPID force=true || echo ERROR
sleep 20
xe snapshot-uninstall uuid=$SNAPID force=true > /dev/null 2>&1 || true
echo "$(date +'%Y-%m-%d %H:%M:%S') Done Backup of vm $VMNAME"