Hi there.
Given is a community edition XO on commit 3500d.
Since June 6th one out of seven VMs is no longer backing up.
It is always the same VM which is failing.
The remote is working (six VMs back up without issue).
[image: 1780938331811-remote_status.png]
It seems XO has trouble cleaning up the backup directory:
[image: 1780938453871-backup_log.png]
Here is the full backup log:
{
"data": {
"mode": "delta",
"reportWhen": "failure"
},
"id": "1780936394962",
"jobId": "21bbca2a-59ad-4fa2-bb73-f60c1e9216e7",
"jobName": "VM-Backup",
"message": "backup",
"scheduleId": "15044139-f4b7-4078-92bc-891e9b641d52",
"start": 1780936394962,
"status": "failure",
"tasks": [
{
"id": "0mq5fjqwf-22j12nbve5e",
"start": 1780936397295,
"status": "failure",
"tasks": [
{
"id": "0mq5fjqwy-ymsrhlbl67s",
"start": 1780936397314,
"status": "success",
"warnings": [
{
"data": {
"path": "/xo-vm-backups/011114a9-b622-92eb-581f-36f961efe622/vdis/21bbca2a-59ad-4fa2-bb73-f60c1e9216e7/63a73be5-96ff-4c25-b7dd-c93284719916/20260608T150007Z.vhd",
"error": {}
},
"message": "failed to read disk parent info"
}
],
"end": 1780936399044,
"result": {
"merge": false,
"size": 0
},
"message": "clean-vm"
},
{
"id": "0mq5fku8o-i3wwl2zfkza",
"start": 1780936448280,
"status": "success",
"warnings": [
{
"data": {
"path": "/xo-vm-backups/011114a9-b622-92eb-581f-36f961efe622/vdis/21bbca2a-59ad-4fa2-bb73-f60c1e9216e7/63a73be5-96ff-4c25-b7dd-c93284719916/20260608T150007Z.vhd",
"error": {}
},
"message": "failed to read disk parent info"
}
],
"end": 1780936454149,
"result": {
"merge": false,
"size": 0
},
"message": "clean-vm"
}
],
"end": 1780936454187,
"result": {
"code": "VDI_IN_USE",
"params": [
"OpaqueRef:ce1e2c1b-9b52-d813-88fc-18513a51306f",
"destroy"
],
"task": {
"uuid": "173eb901-7e4b-90d5-48cb-1306493d61d5",
"name_label": "Async.VDI.destroy",
"name_description": "",
"allowed_operations": [],
"current_operations": {},
"created": "20260608T16:34:06Z",
"finished": "20260608T16:34:08Z",
"status": "failure",
"resident_on": "OpaqueRef:8a7ca417-3c20-466f-8ebc-f19cbb63384f",
"progress": 1,
"type": "<none/>",
"result": "",
"error_info": [
"VDI_IN_USE",
"OpaqueRef:ce1e2c1b-9b52-d813-88fc-18513a51306f",
"destroy"
],
"other_config": {},
"subtask_of": "OpaqueRef:NULL",
"subtasks": [],
"backtrace": "(((process xapi)(filename ocaml/xapi/message_forwarding.ml)(line 5212))((process xapi)(filename fun.ml)(line 33))((process xapi)(filename fun.ml)(line 38))((process xapi)(filename ocaml/xapi/helpers.ml)(line 1821))((process xapi)(filename ocaml/xapi/message_forwarding.ml)(line 5201))((process xapi)(filename ocaml/xapi/rbac.ml)(line 228))((process xapi)(filename ocaml/xapi/rbac.ml)(line 238))((process xapi)(filename ocaml/xapi/server_helpers.ml)(line 78)))"
},
"message": "VDI_IN_USE(OpaqueRef:ce1e2c1b-9b52-d813-88fc-18513a51306f, destroy)",
"name": "XapiError",
"stack": "XapiError: VDI_IN_USE(OpaqueRef:ce1e2c1b-9b52-d813-88fc-18513a51306f, destroy)\n at XapiError.wrap (file:///opt/xo/xo-builds/xen-orchestra-202606081816/packages/xen-api/_XapiError.mjs:16:12)\n at default (file:///opt/xo/xo-builds/xen-orchestra-202606081816/packages/xen-api/_getTaskResult.mjs:13:29)\n at Xapi._addRecordToCache (file:///opt/xo/xo-builds/xen-orchestra-202606081816/packages/xen-api/index.mjs:1078:24)\n at file:///opt/xo/xo-builds/xen-orchestra-202606081816/packages/xen-api/index.mjs:1112:14\n at Array.forEach (<anonymous>)\n at Xapi._processEvents (file:///opt/xo/xo-builds/xen-orchestra-202606081816/packages/xen-api/index.mjs:1102:12)\n at Xapi._watchEvents (file:///opt/xo/xo-builds/xen-orchestra-202606081816/packages/xen-api/index.mjs:1275:14)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)"
},
"message": "backup VM",
"data": {
"id": "011114a9-b622-92eb-581f-36f961efe622",
"type": "VM",
"name_label": "ITServices"
}
},
{
"id": "0mq5fjqwh-hhuivj17nmu",
"start": 1780936397297,
"status": "success",
"tasks": [
{
"id": "0mq5fjqx1-nwsciopgbxh",
"start": 1780936397317,
"status": "success",
"end": 1780936399283,
"result": {
"merge": false,
"size": 0
},
"message": "clean-vm"
},
{
"id": "0mq5fjsl1-29b0qivtg74",
"start": 1780936399477,
"status": "success",
"end": 1780936401784,
"result": "0d56ceb1-a627-1c97-e25d-ab641d5d3317",
"message": "snapshot"
},
{
"id": "0mq5fjud4-an4b4yrzygv",
"start": 1780936401784,
"status": "success",
"tasks": [
{
"id": "0mq5fjvy2-qfouxpigsx",
"start": 1780936403834,
"status": "success",
"end": 1780936418474,
"result": {
"size": 652498432
},
"message": "transfer",
"data": {
"progress": 99
}
},
{
"id": "0mq5fkcfx-f456mkr06ov",
"start": 1780936425213,
"status": "success",
"end": 1780936426551,
"result": {
"merge": true,
"size": 0
},
"message": "clean-vm"
}
],
"end": 1780936428639,
"message": "export",
"data": {
"id": "442a4fc2-70ff-474d-aae9-88d0249ef5f2",
"isFull": false,
"type": "remote"
}
}
],
"end": 1780936428639,
"message": "backup VM",
"data": {
"id": "44ae09b4-47d2-ab1c-e187-3020f0626d74",
"type": "VM",
"name_label": "Unifi_OS_Server",
"progress": 0
}
},
{
"id": "0mq5fjqwi-vahgy7faqum",
"start": 1780936397298,
"status": "success",
"tasks": [
{
"id": "0mq5fjqx0-7cex64zgf22",
"start": 1780936397316,
"status": "success",
"end": 1780936399539,
"result": {
"merge": false,
"size": 0
},
"message": "clean-vm"
},
{
"id": "0mq5fjswh-ubn0k70kwtb",
"start": 1780936399889,
"status": "success",
"end": 1780936402821,
"result": "5dc9f1c0-e790-c9cd-1309-ec58f64beedb",
"message": "snapshot"
},
{
"id": "0mq5fjv5x-4cl03s5xrcs",
"start": 1780936402821,
"status": "success",
"tasks": [
{
"id": "0mq5fjwif-1eoxjx0ass2",
"start": 1780936404567,
"status": "success",
"end": 1780936434954,
"result": {
"size": 1021871616
},
"message": "transfer",
"data": {
"progress": 100
}
},
{
"id": "0mq5fkklg-ubob0jzzxb",
"start": 1780936435780,
"status": "success",
"end": 1780936441161,
"result": {
"merge": true,
"size": 0
},
"message": "clean-vm"
}
],
"end": 1780936441176,
"message": "export",
"data": {
"id": "442a4fc2-70ff-474d-aae9-88d0249ef5f2",
"isFull": false,
"type": "remote"
}
}
],
"end": 1780936441176,
"message": "backup VM",
"data": {
"id": "e764ee17-19de-29fc-802f-e97385d0635d",
"type": "VM",
"name_label": "Graylog",
"progress": 0
}
},
{
"id": "0mq5fjqwi-eyuvf4i18bv",
"start": 1780936397298,
"status": "success",
"tasks": [
{
"id": "0mq5fjqwx-kykb7016aud",
"start": 1780936397313,
"status": "success",
"end": 1780936399348,
"result": {
"merge": false,
"size": 0
},
"message": "clean-vm"
},
{
"id": "0mq5fjsl0-1jdpq922616",
"start": 1780936399476,
"status": "success",
"end": 1780936400960,
"result": "d403776f-e797-5af2-2428-35963e1af590",
"message": "snapshot"
},
{
"id": "0mq5fjtq9-o2xr8f7gelf",
"start": 1780936400961,
"status": "success",
"tasks": [
{
"id": "0mq5fjvcs-7jaedl39bym",
"start": 1780936403068,
"status": "success",
"end": 1780936471627,
"result": {
"size": 4130425344
},
"message": "transfer",
"data": {
"progress": 100
}
},
{
"id": "0mq5fld9j-pzmbsqgh56",
"start": 1780936472935,
"status": "success",
"end": 1780936476387,
"result": {
"merge": true,
"size": 0
},
"message": "clean-vm"
}
],
"end": 1780936476416,
"message": "export",
"data": {
"id": "442a4fc2-70ff-474d-aae9-88d0249ef5f2",
"isFull": false,
"type": "remote"
}
}
],
"end": 1780936476416,
"message": "backup VM",
"data": {
"id": "a329942c-10e9-70d4-4155-95c70e6c25c8",
"type": "VM",
"name_label": "XenOrchestra_Prod",
"progress": 0
}
},
{
"id": "0mq5fkf41-weadm4wk1xp",
"start": 1780936428673,
"status": "success",
"tasks": [
{
"id": "0mq5fkf5n-2ponux8v3bm",
"start": 1780936428731,
"status": "success",
"end": 1780936430525,
"result": {
"merge": false,
"size": 0
},
"message": "clean-vm"
},
{
"id": "0mq5fkgor-fwufkt4zm1f",
"start": 1780936430715,
"status": "success",
"end": 1780936431803,
"result": "649ca352-0aab-dbd9-d75b-aae8748fb6a8",
"message": "snapshot"
},
{
"id": "0mq5fkhiz-u5o8qh9lpoh",
"start": 1780936431803,
"status": "success",
"tasks": [
{
"id": "0mq5fklec-1glqm0n1642",
"start": 1780936436820,
"status": "success",
"end": 1780936437167,
"result": {
"size": 43008
},
"message": "transfer",
"data": {
"progress": 100
}
},
{
"id": "0mq5fkn5y-g2hl3nyuc2f",
"start": 1780936439110,
"status": "success",
"end": 1780936440668,
"result": {
"merge": true,
"size": 0
},
"message": "clean-vm"
}
],
"end": 1780936440676,
"message": "export",
"data": {
"id": "442a4fc2-70ff-474d-aae9-88d0249ef5f2",
"isFull": false,
"type": "remote"
}
}
],
"end": 1780936440676,
"message": "backup VM",
"data": {
"id": "8ec416aa-4885-b82b-e9ac-24df17672916",
"type": "VM",
"name_label": "RustDesk",
"progress": 0
}
},
{
"id": "0mq5fkodk-tkwoc2uthf",
"start": 1780936440680,
"status": "success",
"tasks": [
{
"id": "0mq5fkodq-e3kpftr1kr",
"start": 1780936440686,
"status": "success",
"end": 1780936443708,
"result": {
"merge": false,
"size": 0
},
"message": "clean-vm"
},
{
"id": "0mq5fkqvi-egmgr1fdy0t",
"start": 1780936443918,
"status": "success",
"end": 1780936447236,
"result": "8bec4422-5b08-5410-eff2-4f8d8c6d0ac7",
"message": "snapshot"
},
{
"id": "0mq5fktfo-iuhc7r90uvc",
"start": 1780936447236,
"status": "success",
"tasks": [
{
"id": "0mq5fkv4r-dz2833iynb",
"start": 1780936449435,
"status": "success",
"end": 1780936457597,
"result": {
"size": 180524032
},
"message": "transfer",
"data": {
"progress": 100
}
},
{
"id": "0mq5fl3af-hde5wx2577",
"start": 1780936460007,
"status": "success",
"end": 1780936461168,
"result": {
"merge": true,
"size": 0
},
"message": "clean-vm"
}
],
"end": 1780936461396,
"message": "export",
"data": {
"id": "442a4fc2-70ff-474d-aae9-88d0249ef5f2",
"isFull": false,
"type": "remote"
}
}
],
"end": 1780936461396,
"message": "backup VM",
"data": {
"id": "db880663-5c76-ad1d-ce9c-126db11c4822",
"type": "VM",
"name_label": "Nextcloud",
"progress": 0
}
},
{
"id": "0mq5fkorf-psnurqj48qp",
"start": 1780936441179,
"status": "success",
"tasks": [
{
"id": "0mq5fkork-ui1kc8oo0i9",
"start": 1780936441184,
"status": "success",
"end": 1780936443835,
"result": {
"merge": false,
"size": 0
},
"message": "clean-vm"
},
{
"id": "0mq5fkr16-ve61u2lg6yr",
"start": 1780936444122,
"status": "success",
"end": 1780936448319,
"result": "4436636b-5f56-347c-45d9-339dfbde7809",
"message": "snapshot"
},
{
"id": "0mq5fku9r-4i3dvxltfoh",
"start": 1780936448319,
"status": "success",
"tasks": [
{
"id": "0mq5fkuzl-zbg2ht2gowa",
"start": 1780936449249,
"status": "success",
"end": 1780936568326,
"result": {
"size": 8392878592
},
"message": "transfer",
"data": {
"progress": 99
}
},
{
"id": "0mq5fnfo4-lh7uyjgbpw",
"start": 1780936569364,
"status": "success",
"end": 1780936570543,
"result": {
"merge": true,
"size": 0
},
"message": "clean-vm"
}
],
"end": 1780936570549,
"message": "export",
"data": {
"id": "442a4fc2-70ff-474d-aae9-88d0249ef5f2",
"isFull": true,
"type": "remote"
}
}
],
"end": 1780936570549,
"message": "backup VM",
"data": {
"id": "d01f5065-7ce2-cdfe-a11d-066000777c63",
"type": "VM",
"name_label": "Netbox",
"progress": 0
}
}
],
"end": 1780936570551,
"infos": [
{
"data": {
"vms": [
"44ae09b4-47d2-ab1c-e187-3020f0626d74",
"e764ee17-19de-29fc-802f-e97385d0635d",
"011114a9-b622-92eb-581f-36f961efe622",
"a329942c-10e9-70d4-4155-95c70e6c25c8",
"8ec416aa-4885-b82b-e9ac-24df17672916",
"db880663-5c76-ad1d-ce9c-126db11c4822",
"d01f5065-7ce2-cdfe-a11d-066000777c63"
]
},
"message": "vms"
}
]
}
Maybe this is linked to the thread by @andrew :
https://xcp-ng.org/forum/topic/12249/xo-error-warning-clean-vm-directory.-unhandled-error-while-checking-alias./37
I upgraded today to commit 3500d, rebooted the XO VM and retried the backup. Still failing.
Thanks for the assistance.
Regards,
Marc