I'm getting an increasing number of random backup failures lately, and almost always at the same step. They almost always fail at the health check while waiting for the VM to start. The storage is NFS on a server that has a 10g connection, same as the production storage so it should be fine, not detecting issues with the storage used for backup so not sure what's going on. Here is the short log of one of these, storage reports no errors on the Truenas server.
{
"data": {
"mode": "full",
"reportWhen": "failure"
},
"id": "1792559700001",
"jobId": "05f70792-9624-4150-9597-5917c045bea6",
"jobName": "DC2-backup",
"message": "backup",
"scheduleId": "5ec0bd0c-b537-4da9-afff-3f1eb0fbd3e9",
"start": 1792559700001,
"status": "failure",
"tasks": [
{
"id": "0mvhnriw7-cddb6rumww7",
"start": 1792559702887,
"status": "failure",
"tasks": [
{
"id": "0mvhnrjkn-ibpty24qqpg",
"start": 1792559703767,
"status": "success",
"end": 1792559705242,
"result": "05a07f3b-2c62-13ef-3405-17166901cefb",
"message": "snapshot"
},
{
"id": "0mvhnrkq7-v610f7ynxw",
"start": 1792559705263,
"status": "success",
"tasks": [
{
"id": "0mvhnrkrm-un28zyxt91k",
"start": 1792559705314,
"status": "success",
"end": 1792560387346,
"result": {
"size": 153946460672
},
"message": "transfer",
"data": {
"progress": 0
}
}
],
"end": 1792560387560,
"message": "export",
"data": {
"id": "0c6ed546-4ff5-49b8-b28e-80bbd3513e52",
"type": "remote",
"isFull": true
}
},
{
"id": "0mvho67u3-ty880j4lxuk",
"start": 1792560388395,
"status": "failure",
"tasks": [
{
"id": "0mvho67w9-okbayxed8k",
"start": 1792560388473,
"status": "success",
"end": 1792560962736,
"result": {
"size": 153946460672,
"id": "4ac820a4-133b-e314-a6fd-d1817051ccad",
"liveMounts": []
},
"message": "transfer",
"data": {
"progress": 100
}
},
{
"id": "0mvhoij02-se35olnw9kf",
"start": 1792560962738,
"status": "failure",
"end": 1792561562889,
"result": {
"message": "timeout reached while waiting for OpaqueRef:a451ca09-fb0f-06c1-5ee5-70cd22f9fc51 to report the driver version through the Xen tools. Please check or update the Xen tools.",
"name": "Error",
"stack": "Error: timeout reached while waiting for OpaqueRef:a451ca09-fb0f-06c1-5ee5-70cd22f9fc51 to report the driver version through the Xen tools. Please check or update the Xen tools.\n at file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/xapi/index.mjs:295:23\n at new Promise (<anonymous>)\n at Xapi.waitObjectState (file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/xapi/index.mjs:279:12)\n at file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/backups/HealthCheckVmBackup.mjs:63:20\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async Task.runInside (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:204:22)\n at async Task.run (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:188:20)\n at async file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/backups/_runners/_writers/_MixinRemoteWriter.mjs:126:13\n at async Task.runInside (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:204:22)\n at async Task.run (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:188:20)"
},
"message": "vmstart"
}
],
"end": 1792561565797,
"result": {
"message": "timeout reached while waiting for OpaqueRef:a451ca09-fb0f-06c1-5ee5-70cd22f9fc51 to report the driver version through the Xen tools. Please check or update the Xen tools.",
"name": "Error",
"stack": "Error: timeout reached while waiting for OpaqueRef:a451ca09-fb0f-06c1-5ee5-70cd22f9fc51 to report the driver version through the Xen tools. Please check or update the Xen tools.\n at file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/xapi/index.mjs:295:23\n at new Promise (<anonymous>)\n at Xapi.waitObjectState (file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/xapi/index.mjs:279:12)\n at file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/backups/HealthCheckVmBackup.mjs:63:20\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async Task.runInside (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:204:22)\n at async Task.run (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:188:20)\n at async file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/backups/_runners/_writers/_MixinRemoteWriter.mjs:126:13\n at async Task.runInside (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:204:22)\n at async Task.run (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:188:20)"
},
"message": "health check",
"data": {
"progress": 0
}
},
{
"id": "0mvhovgc2-ih7l8hmrak",
"start": 1792561565810,
"status": "success",
"end": 1792561567438,
"result": {
"removedFiles": [],
"changedFiles": [],
"merge": false,
"size": 0
},
"message": "clean-vm"
}
],
"end": 1792561567439,
"result": {
"message": "timeout reached while waiting for OpaqueRef:a451ca09-fb0f-06c1-5ee5-70cd22f9fc51 to report the driver version through the Xen tools. Please check or update the Xen tools.",
"name": "Error",
"stack": "Error: timeout reached while waiting for OpaqueRef:a451ca09-fb0f-06c1-5ee5-70cd22f9fc51 to report the driver version through the Xen tools. Please check or update the Xen tools.\n at file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/xapi/index.mjs:295:23\n at new Promise (<anonymous>)\n at Xapi.waitObjectState (file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/xapi/index.mjs:279:12)\n at file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/backups/HealthCheckVmBackup.mjs:63:20\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async Task.runInside (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:204:22)\n at async Task.run (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:188:20)\n at async file:///opt/xo/xo-builds/xen-orchestra-202610130947/@xen-orchestra/backups/_runners/_writers/_MixinRemoteWriter.mjs:126:13\n at async Task.runInside (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:204:22)\n at async Task.run (/opt/xo/xo-builds/xen-orchestra-202610130947/@vates/task/index.js:188:20)"
},
"message": "backup VM",
"data": {
"id": "1c163183-d863-f8cc-159a-5c5281aafee3",
"type": "VM",
"name_label": "BMC-DC2",
"progress": 0
}
}
],
"end": 1792561567595,
"infos": [
{
"data": {
"vms": [
"1c163183-d863-f8cc-159a-5c5281aafee3"
]
},
"message": "vms"
}
]
}