I'm getting an increasing number of random backup failures lately, and almost always at the same step. They almost always fail at the health check while waiting for the VM to start. The storage is NFS on a server that has a 10g connection, same as the production storage so it should be fine, not detecting issues with the storage used for backup so not sure what's going on. Here is the short log of one of these, storage reports no errors on the Truenas server.
XCP-NG and XO (sources) are fairly up to date, I think XO was updated about a week ago and XCP is current patch level. I had a different backup job that was routinely failing, but I think it was happening when a scub in the storage was happening. Changed the time and it was OK after that.
{
"data": {
"mode": "full",
"reportWhen": "failure"
},
"id": "1792559700001",
"jobId": "05f70792-9624-4150-9597-5917c045bea6",
"jobName": "DC2-backup",
"message": "backup",
"scheduleId": "5ec0bd0c-b537-4da9-afff-3f1eb0fbd3e9",
"start": 1792559700001,
"status": "failure",
"tasks": [
{
"id": "0mvhnriw7-cddb6rumww7",
"start": 1792559702887,
"status": "failure",
"tasks": [
{
"id": "0mvhnrjkn-ibpty24qqpg",
"start": 1792559703767,
"status": "success",
"end": 1792559705242,
"result": "05a07f3b-2c62-13ef-3405-17166901cefb",
"message": "snapshot"
},
{
"id": "0mvhnrkq7-v610f7ynxw",
"start": 1792559705263,
"status": "success",
"tasks": [
{
"id": "0mvhnrkrm-un28zyxt91k",
"start": 1792559705314,
"status": "success",
"end": 1792560387346,
"result": {
"size": 153946460672
},
"message": "transfer",
"data": {
"progress": 0
}
}
],
"end": 1792560387560,
"message": "export",
"data": {
"id": "0c6ed546-4ff5-49b8-b28e-80bbd3513e52",
"type": "remote",
"isFull": true
}
},
{
"id": "0mvho67u3-ty880j4lxuk",
"start": 1792560388395,
"status": "failure",
"tasks": [
{
"id": "0mvho67w9-okbayxed8k",
"start": 1792560388473,
"status": "success",
"end": 1792560962736,
"result": {
"size": 153946460672,
"id": "4ac820a4-133b-e314-a6fd-d1817051ccad",
"liveMounts": []
},
"message": "transfer",
"data": {
"progress": 100
}
},
{
"id": "0mvhoij02-se35olnw9kf",
"start": 1792560962738,
"status": "failure",
"end": 1792561562889,
"result": {
"message": "timeout reached while waiting for OpaqueRef:a451ca09-fb0f-06c1-5ee5-70cd22f9fc51 to report the driver version through the Xen tools. Please check or update the Xen tools.",
"name": "Error",
"stack": "Error: timeout reached while waiting for OpaqueRef:a451ca09-fb0f-06c1-5ee5-70cd22f9fc51 to report the driver version through the Xen tools. Please check or update the Xen tools.\n at file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/xapi/index.mjs:295:23\n at new Promise (<anonymous>)\n at Xapi.waitObjectState (file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/xapi/index.mjs:279:12)\n at file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/backups/HealthCheckVmBackup.mjs:63:20\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async Task.runInside (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:204:22)\n at async Task.run (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:188:20)\n at async file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/backups/_runners/_writers/_MixinRemoteWriter.mjs:126:13\n at async Task.runInside (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:204:22)\n at async Task.run (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:188:20)"
},
"message": "vmstart"
}
],
"end": 1792561565797,
"result": {
"message": "timeout reached while waiting for OpaqueRef:a451ca09-fb0f-06c1-5ee5-70cd22f9fc51 to report the driver version through the Xen tools. Please check or update the Xen tools.",
"name": "Error",
"stack": "Error: timeout reached while waiting for OpaqueRef:a451ca09-fb0f-06c1-5ee5-70cd22f9fc51 to report the driver version through the Xen tools. Please check or update the Xen tools.\n at file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/xapi/index.mjs:295:23\n at new Promise (<anonymous>)\n at Xapi.waitObjectState (file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/xapi/index.mjs:279:12)\n at file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/backups/HealthCheckVmBackup.mjs:63:20\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async Task.runInside (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:204:22)\n at async Task.run (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:188:20)\n at async file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/backups/_runners/_writers/_MixinRemoteWriter.mjs:126:13\n at async Task.runInside (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:204:22)\n at async Task.run (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:188:20)"
},
"message": "health check",
"data": {
"progress": 0
}
},
{
"id": "0mvhovgc2-ih7l8hmrak",
"start": 1792561565810,
"status": "success",
"end": 1792561567438,
"result": {
"removedFiles": [],
"changedFiles": [],
"merge": false,
"size": 0
},
"message": "clean-vm"
}
],
"end": 1792561567439,
"result": {
"message": "timeout reached while waiting for OpaqueRef:a451ca09-fb0f-06c1-5ee5-70cd22f9fc51 to report the driver version through the Xen tools. Please check or update the Xen tools.",
"name": "Error",
"stack": "Error: timeout reached while waiting for OpaqueRef:a451ca09-fb0f-06c1-5ee5-70cd22f9fc51 to report the driver version through the Xen tools. Please check or update the Xen tools.\n at file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/xapi/index.mjs:295:23\n at new Promise (<anonymous>)\n at Xapi.waitObjectState (file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/xapi/index.mjs:279:12)\n at file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/backups/HealthCheckVmBackup.mjs:63:20\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async Task.runInside (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:204:22)\n at async Task.run (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:188:20)\n at async file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/backups/_runners/_writers/_MixinRemoteWriter.mjs:126:13\n at async Task.runInside (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:204:22)\n at async Task.run (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:188:20)"
},
"message": "backup VM",
"data": {
"id": "1c163183-d863-f8cc-159a-5c5281aafee3",
"type": "VM",
"name_label": "BMC-DC2",
"progress": 0
}
}
],
"end": 1792561567595,
"infos": [
{
"data": {
"vms": [
"1c163183-d863-f8cc-159a-5c5281aafee3"
]
},
"message": "vms"
}
]
}
