xe vdi-forget uuid=c6853f48-4b06-4c34-9707-b68f9054e6fc
xe sr-scan uuid=6b24cd1c-22ad-0994-5b6b-a75389a6ddba
xe vdi-list uuid=c6853f48-4b06-4c34-9707-b68f9054e6fc params=sr-uuid sr-uuid ( RO) : 6b24cd1c-22ad-0994-5b6b-a75389a6ddba
yup, that worked, thanks 
https://xcp-ng.org/docs/api.html
Restarting the API
Sometimes, you might want to restart the toolstack (API and storage stack). You can do it with the following command from the host console (or directly with Xen Orchestra):
xe-toolstack-restart
TIP
Restarting XAPI won't affect any running VMs. However, all backup/export tasks will be removed.
I believe XO 6 is only available for XCP-ng 8.3, not 8.2.
If you are using Ronivay script for building XO, XO 6 Lite via XO (/v6) is disabled by default since about a month ago.
You need to specifically enable XO 6 lite installation in the configuration file included in Ronivays script.
xe vdi-forget uuid=c6853f48-4b06-4c34-9707-b68f9054e6fc
xe sr-scan uuid=6b24cd1c-22ad-0994-5b6b-a75389a6ddba
xe vdi-list uuid=c6853f48-4b06-4c34-9707-b68f9054e6fc params=sr-uuid sr-uuid ( RO) : 6b24cd1c-22ad-0994-5b6b-a75389a6ddba
yup, that worked, thanks
Ok i managed to restore all .vhd files from truenas snapshots, but still cannot start the VM, because one of the disks is stuck on dom0:
vm.start
{
"id": "d5d0334c-a7e3-b29f-51ca-1be9c211d2c1",
"bypassMacAddressesCheck": false,
"force": false
}
{
"code": "SR_BACKEND_FAILURE_46",
"params": [
"",
"The VDI is not available [opterr=VDI 9f8da2fd-a08d-43c2-a5b1-ce6125cf52f5 not detached cleanly]",
""
],
"call": {
"duration": 4622,
"method": "VM.start",
"params": [
"* session id *",
"OpaqueRef:cd8f9bbd-de5e-49a4-bbc0-38abdff5f1ac",
false,
false
]
},
"message": "SR_BACKEND_FAILURE_46(, The VDI is not available [opterr=VDI 9f8da2fd-a08d-43c2-a5b1-ce6125cf52f5 not detached cleanly], )",
"name": "XapiError",
"stack": "XapiError: SR_BACKEND_FAILURE_46(, The VDI is not available [opterr=VDI 9f8da2fd-a08d-43c2-a5b1-ce6125cf52f5 not detached cleanly], )
at Function.wrap (file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/_XapiError.mjs:16:12)
at file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/transports/json-rpc.mjs:38:21
at runNextTicks (node:internal/process/task_queues:65:5)
at processImmediate (node:internal/timers:453:9)
at process.callbackTrampoline (node:internal/async_hooks:130:17)"
}
Tried manually unplugging:
xe vbd-unplug uuid=60c80d37-351c-22e0-81c6-bae24082f4e1 --force
The server failed to handle your request, due to an internal error. The given message may give details useful for debugging the problem.
message: Expected 0 or 1 VDI with datapath, had 2
Already tried restarting toolstack, didnt help.
UPDATE:
I managed to find the missing VDI "c6853f48-4b06-4c34-9707-b68f9054e6fc", it was the last successfull backup taken i believe, luckily i have snapshots on backup folder aswell on truenas!
root@truenas01[.../d5d0334c-a7e3-b29f-51ca-1be9c211d2c1]# ll
total 341
drwxr-xr-x 3 root 19 Jun 7 08:35 ./
drwxr-xr-x 31 nobody 31 Jun 7 09:08 ../
-rw-r--r-- 1 root 20305 Oct 7 2024 20241007T113432Z.json
-rw-r--r-- 1 root 20131 May 27 10:51 20250527T065026Z.json
-rw-r--r-- 1 root 20327 May 28 09:13 20250528T050955Z.json
-rw-r--r-- 1 root 20327 May 28 13:18 20250528T091600Z.json
-rw-r--r-- 1 root 20440 May 29 08:35 20250529T044854Z.json
-rw-r--r-- 1 root 20327 May 30 09:13 20250530T051201Z.json
-rw-r--r-- 1 root 20131 May 30 10:43 20250530T064322Z.json
-rw-r--r-- 1 root 20327 May 31 09:13 20250531T051135Z.json
-rw-r--r-- 1 root 20327 Jun 1 09:13 20250601T051144Z.json
-rw-r--r-- 1 root 20327 Jun 2 09:12 20250602T051127Z.json
-rw-r--r-- 1 root 20327 Jun 3 09:17 20250603T050936Z.json
-rw-r--r-- 1 root 20327 Jun 4 09:07 20250604T050627Z.json
-rw-r--r-- 1 root 20327 Jun 5 09:07 20250605T050623Z.json
-rw-r--r-- 1 root 20327 Jun 6 09:07 20250606T050650Z.json
-rw-r--r-- 1 root 20327 Jun 7 09:08 20250607T050648Z.json
-rw-r--r-- 1 root 17855 Jun 7 09:08 cache.json.gz
drwx------ 5 root 5 Oct 7 2024 vdis/
root@truenas01[.../d5d0334c-a7e3-b29f-51ca-1be9c211d2c1]# cd vdis
root@truenas01[...334c-a7e3-b29f-51ca-1be9c211d2c1/vdis]# cd 5a7a5647-d2ca-4014-b87b-06c621c09a20
root@truenas01[.../5a7a5647-d2ca-4014-b87b-06c621c09a20]# ls
9f8da2fd-a08d-43c2-a5b1-ce6125cf52f5 c6853f48-4b06-4c34-9707-b68f9054e6fc
root@truenas01[.../5a7a5647-d2ca-4014-b87b-06c621c09a20]# cd c6853f48-4b06-4c34-9707-b68f9054e6fc
root@truenas01[.../c6853f48-4b06-4c34-9707-b68f9054e6fc]# ll -hl
total 21G
drwx------ 2 root 3 Oct 7 2024 ./
drwx------ 4 root 4 Oct 7 2024 ../
-rw-r--r-- 1 root 50G Oct 7 2024 20241007T113432Z.vhd
Do i only need to resotre this: 20241007T113432Z.vhd and the VM should be back, since the basedisk still exists or?
Hi,
little over a week ago, one of my VM's backup started to have some hickups after i accidentally rebooted XO (or was it toolstack restart, cant remember) during backups...
After that hickup my delta backup started throwing this error:
Error: The value of "offset" is out of range. It must be >= 0 and <= 102399. Received 102400
I kept getting this error everyday, so i tried resetting the backups by removing the snapshot that existed.
Removing snapshot did not help so i disabled CBT and removed snapshot and all existing delta backups as well, waited for coalescence then tried backup and got this error:
Error: SR_BACKEND_FAILURE_82(, Failed to snapshot VDI [opterr=['MAP_DUPLICATE_KEY', 'VDI', 'sm_config', 'OpaqueRef:2ebd2753-709c-4bf9-afdb-9ce38938b5cf', 'paused']], )
After this i tried migrating the VM to another host and got error:
vm.migrate
{
"vm": "d5d0334c-a7e3-b29f-51ca-1be9c211d2c1",
"migrationNetwork": "1f6f4495-1045-6fe2-3da6-4e43862e623d",
"sr": "6b24cd1c-22ad-0994-5b6b-a75389a6ddba",
"targetHost": "d09b0fef-7aab-4516-a21e-0f72806655aa"
}
{
"code": "SR_BACKEND_FAILURE_202",
"params": [
"",
"General backend error [opterr=['MAP_DUPLICATE_KEY', 'VDI', 'sm_config', 'OpaqueRef:2ebd2753-709c-4bf9-afdb-9ce38938b5cf', 'paused']]",
""
],
"task": {
"uuid": "1d7d6b6c-1d5d-f166-c1e9-d42a94a08253",
"name_label": "Async.VDI.disable_cbt",
"name_description": "",
"allowed_operations": [],
"current_operations": {},
"created": "20250613T08:15:55Z",
"finished": "20250613T08:15:56Z",
"status": "failure",
"resident_on": "OpaqueRef:010eebba-be27-489f-9f87-d06c8b675f19",
"progress": 1,
"type": "<none/>",
"result": "",
"error_info": [
"SR_BACKEND_FAILURE_202",
"",
"General backend error [opterr=['MAP_DUPLICATE_KEY', 'VDI', 'sm_config', 'OpaqueRef:2ebd2753-709c-4bf9-afdb-9ce38938b5cf', 'paused']]",
""
],
"other_config": {},
"subtask_of": "OpaqueRef:NULL",
"subtasks": [],
"backtrace": "(((process xapi)(filename lib/backtrace.ml)(line 210))((process xapi)(filename ocaml/xapi/storage_access.ml)(line 32))((process xapi)(filename ocaml/xapi/xapi_vdi.ml)(line 1403))((process xapi)(filename ocaml/xapi/message_forwarding.ml)(line 131))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename ocaml/xapi/rbac.ml)(line 205))((process xapi)(filename ocaml/xapi/server_helpers.ml)(line 95)))"
},
"message": "SR_BACKEND_FAILURE_202(, General backend error [opterr=['MAP_DUPLICATE_KEY', 'VDI', 'sm_config', 'OpaqueRef:2ebd2753-709c-4bf9-afdb-9ce38938b5cf', 'paused']], )",
"name": "XapiError",
"stack": "XapiError: SR_BACKEND_FAILURE_202(, General backend error [opterr=['MAP_DUPLICATE_KEY', 'VDI', 'sm_config', 'OpaqueRef:2ebd2753-709c-4bf9-afdb-9ce38938b5cf', 'paused']], )
at Function.wrap (file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/_XapiError.mjs:16:12)
at default (file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/_getTaskResult.mjs:13:29)
at Xapi._addRecordToCache (file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/index.mjs:1072:24)
at file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/index.mjs:1106:14
at Array.forEach (<anonymous>)
at Xapi._processEvents (file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/index.mjs:1096:12)
at Xapi._watchEvents (file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/index.mjs:1269:14)"
}
I then tried restarting the host and got error:
vm.restart
{
"id": "d5d0334c-a7e3-b29f-51ca-1be9c211d2c1",
"force": true,
"bypassBlockedOperation": false
}
{
"code": "SR_BACKEND_FAILURE_46",
"params": [
"",
"The VDI is not available [opterr=VDI c6853f48-4b06-4c34-9707-b68f9054e6fc locked]",
""
],
"task": {
"uuid": "421e7331-322d-5db1-02eb-62a6e443f4e7",
"name_label": "Async.VM.hard_reboot",
"name_description": "",
"allowed_operations": [],
"current_operations": {},
"created": "20250613T08:27:36Z",
"finished": "20250613T08:31:42Z",
"status": "failure",
"resident_on": "OpaqueRef:010eebba-be27-489f-9f87-d06c8b675f19",
"progress": 1,
"type": "<none/>",
"result": "",
"error_info": [
"SR_BACKEND_FAILURE_46",
"",
"The VDI is not available [opterr=VDI c6853f48-4b06-4c34-9707-b68f9054e6fc locked]",
""
],
"other_config": {},
"subtask_of": "OpaqueRef:NULL",
"subtasks": [],
"backtrace": "(((process xapi)(filename ocaml/xapi-client/client.ml)(line 7))((process xapi)(filename ocaml/xapi-client/client.ml)(line 19))((process xapi)(filename ocaml/xapi-client/client.ml)(line 6122))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 35))((process xapi)(filename ocaml/xapi/message_forwarding.ml)(line 134))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 35))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename ocaml/xapi/rbac.ml)(line 205))((process xapi)(filename ocaml/xapi/server_helpers.ml)(line 95)))"
},
"message": "SR_BACKEND_FAILURE_46(, The VDI is not available [opterr=VDI c6853f48-4b06-4c34-9707-b68f9054e6fc locked], )",
"name": "XapiError",
"stack": "XapiError: SR_BACKEND_FAILURE_46(, The VDI is not available [opterr=VDI c6853f48-4b06-4c34-9707-b68f9054e6fc locked], )
at Function.wrap (file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/_XapiError.mjs:16:12)
at default (file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/_getTaskResult.mjs:13:29)
at Xapi._addRecordToCache (file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/index.mjs:1072:24)
at file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/index.mjs:1106:14
at Array.forEach (<anonymous>)
at Xapi._processEvents (file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/index.mjs:1096:12)
at Xapi._watchEvents (file:///opt/xo/xo-builds/xen-orchestra-202506030701/packages/xen-api/index.mjs:1269:14)"
}
And i then noticed that the VM is actually frozen, the VDI is not found, anywhere. and obviously the VM wont start...
I then noticed in storage > Advance tab this error:
Failed to unpause tapdisk for VDI c6853f48-4b06-4c34-9707-b68f9054e6fc, VMs using this tapdisk have lost access to the corresponding disk(s)
I see the disks in XO and in CLI:
xe vdi-list uuid=c6853f48-4b06-4c34-9707-b68f9054e6fc
uuid ( RO) : c6853f48-4b06-4c34-9707-b68f9054e6fc
name-label ( RW): vm_disk1
name-description ( RW): xvda1
sr-uuid ( RO): 6b24cd1c-22ad-0994-5b6b-a75389a6ddba
virtual-size ( RO): 53689188352
sharable ( RO): false
read-only ( RO): false
But when i check my storage repository in /var/run/sr-mount/ the file is not actually there:
file c6853f48-4b06-4c34-9707-b68f9054e6fc
c6853f48-4b06-4c34-9707-b68f9054e6fc: cannot open (No such file or directory)
Since i just deleted all backup before i began this operation, my only hope is that the "Base copy" VDI which still exists, allows me to recover the VM.
file 3507a5eb-748b-4d13-bdc8-e8128da0bfdb.vhd
3507a5eb-748b-4d13-bdc8-e8128da0bfdb.vhd: Microsoft Disk Image, Virtual Server or Virtual PC
Is this my only hope or recovering the disk or is there some other way i have missed?
I believe XO 6 is only available for XCP-ng 8.3, not 8.2.
If you are using Ronivay script for building XO, XO 6 Lite via XO (/v6) is disabled by default since about a month ago.
You need to specifically enable XO 6 lite installation in the configuration file included in Ronivays script.
Hi,
i tried running Rolling pool update for my servers today, first host was correctly updated and rebooted, but the process got stuck on 2nd host.
After investigating, i found that one of my VM's was not being migrated from the host, which is why the update process failed.
Initial Rolling pool update error was this:
pool.rollingUpdate
{
"pool": "fe688bb2-b9ac-db7b-737a-cc457195f095"
}
{
"code": "VM_SUSPEND_TIMEOUT",
"params": [
"OpaqueRef:5c1818ff-cb37-4103-993a-4b80fa8c8231",
"1200."
],
"task": {
"uuid": "4547170a-ec23-4ad7-128c-69958985de34
",
"name_label": "Async.host.evacuate",
"name_description": "",
"allowed_operations": [],
"current_operations": {},
"created": "20250210T12:37:20Z",
"finished": "20250210T12:58:42Z",
"status": "failure",
"resident_on": "OpaqueRef:010eebba-be27-489f-9f87-d06c8b675f19",
"progress": 1,
"type": "<none/>",
"result": "",
"error_info": [
"VM_SUSPEND_TIMEOUT",
"OpaqueRef:5c1818ff-cb37-4103-993a-4b80fa8c8231",
"1200."
],
"other_config": {},
"subtask_of": "OpaqueRef:NULL",
"subtasks": [],
"backtrace": "(((process xapi)(filename ocaml/xapi-client/client.ml)(line 7))((process xapi)(filename ocaml/xapi-client/client.ml)(line 19))((process xapi)(filename ocaml/xapi-client/client.ml)(line 6172))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 35))((process xapi)(filename ocaml/xapi/xapi_host.ml)(line 612))((process xapi)(filename ocaml/xapi/xapi_host.ml)(line 621))((process xapi)(filename hashtbl.ml)(line 266))((process xapi)(filename hashtbl.ml)(line 272))((process xapi)(filename hashtbl.ml)(line 277))((process xapi)(filename ocaml/xapi/xapi_host.ml)(line 629))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename ocaml/xapi/rbac.ml)(line 205))((process xapi)(filename ocaml/xapi/server_helpers.ml)(line 95)))"
},
"message": "VM_SUSPEND_TIMEOUT(OpaqueRef:5c1818ff-cb37-4103-993a-4b80fa8c8231, 1200.)",
"name": "XapiError",
"stack": "XapiError: VM_SUSPEND_TIMEOUT(OpaqueRef:5c1818ff-cb37-4103-993a-4b80fa8c8231, 1200.)
at Function.wrap (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/_XapiError.mjs:16:12)
at default (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/_getTaskResult.mjs:13:29)
at Xapi._addRecordToCache (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1041:24)
at file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1075:14
at Array.forEach (<anonymous>)
at Xapi._processEvents (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1065:12)
at Xapi._watchEvents (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1238:14)"
}
I then tried manually migrating which failed:
vm.migrate
{
"vm": "0c012493-da75-832d-3f3a-cadce8afb757",
"migrationNetwork": "1f6f4495-1045-6fe2-3da6-4e43862e623d",
"sr": "6b24cd1c-22ad-0994-5b6b-a75389a6ddba",
"targetHost": "48bf1075-066f-4ed1-ba54-a350da4a426c"
}
{
"code": "INTERNAL_ERROR",
"params": [
"Storage_error ([S(Internal_error);S(Xmlrpc_client.Connection_reset)])"
],
"task": {
"uuid": "92d36644-b919-fdf3-e879-475be5b4d8c9",
"name_label": "Async.VM.migrate_send",
"name_description": "",
"allowed_operations": [],
"current_operations": {},
"created": "20250210T13:40:48Z",
"finished": "20250210T13:40:54Z",
"status": "failure",
"resident_on": "OpaqueRef:010eebba-be27-489f-9f87-d06c8b675f19",
"progress": 1,
"type": "<none/>",
"result": "",
"error_info": [
"INTERNAL_ERROR",
"Storage_error ([S(Internal_error);S(Xmlrpc_client.Connection_reset)])"
],
"other_config": {},
"subtask_of": "OpaqueRef:NULL",
"subtasks": [],
"backtrace": "(((process xapi)(filename ocaml/xapi/helpers.ml)(line 1690))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 35))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 35))((process xapi)(filename ocaml/xapi/message_forwarding.ml)(line 134))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 35))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename ocaml/xapi/rbac.ml)(line 205))((process xapi)(filename ocaml/xapi/server_helpers.ml)(line 95)))"
},
"message": "INTERNAL_ERROR(Storage_error ([S(Internal_error);S(Xmlrpc_client.Connection_reset)]))",
"name": "XapiError",
"stack": "XapiError: INTERNAL_ERROR(Storage_error ([S(Internal_error);S(Xmlrpc_client.Connection_reset)]))
at Function.wrap (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/_XapiError.mjs:16:12)
at default (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/_getTaskResult.mjs:13:29)
at Xapi._addRecordToCache (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1041:24)
at file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1075:14
at Array.forEach (<anonymous>)
at Xapi._processEvents (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1065:12)
at Xapi._watchEvents (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1238:14)"
}
I also tried Warm migration, that job was just "Interrupted" found no log for it.
After little digging i found that issue might actually be in the VM that was being migrated, i found this in its kern.log:
Feb 10 13:58:34 lapio kernel: [9681908.582545] Freezing user space processes ...
Feb 10 13:58:34 lapio kernel: [9681928.590134] Freezing of tasks failed after 20.007 seconds (1 tasks refusing to freeze, wq_busy=0):
Feb 10 13:58:34 lapio kernel: [9681928.590463] tar D ffff880009a7b978 0 26564 1 0x00000006
Feb 10 13:58:34 lapio kernel: [9681928.590469] ffff880009a7b978 ffff880120d0a800 ffff88020444c740 ffff8802012f3900
Feb 10 13:58:34 lapio kernel: [9681928.590473] ffff880009a7c000 ffff880205f162c0 7fffffffffffffff ffffffff81867910
Feb 10 13:58:34 lapio kernel: [9681928.590477] ffff880009a7bab8 ffff880009a7b990 ffffffff81867065 0000000000000000
Feb 10 13:58:34 lapio kernel: [9681928.590481] Call Trace:
Feb 10 13:58:34 lapio kernel: [9681928.590494] [<ffffffff81867910>] ? bit_wait+0x60/0x60
Feb 10 13:58:34 lapio kernel: [9681928.590499] [<ffffffff81867065>] schedule+0x35/0x80
Feb 10 13:58:34 lapio kernel: [9681928.590503] [<ffffffff8186a7d8>] schedule_timeout+0x208/0x280
Feb 10 13:58:34 lapio kernel: [9681928.590528] [<ffffffffc050ea49>] ? cifsFileInfo_put+0xa9/0x3f0 [cifs]
Feb 10 13:58:34 lapio kernel: [9681928.590534] [<ffffffff81023d95>] ? xen_clocksource_get_cycles+0x15/0x20
Feb 10 13:58:34 lapio kernel: [9681928.590538] [<ffffffff81867910>] ? bit_wait+0x60/0x60
Feb 10 13:58:34 lapio kernel: [9681928.590542] [<ffffffff818667b4>] io_schedule_timeout+0xa4/0x110
Feb 10 13:58:34 lapio kernel: [9681928.590546] [<ffffffff8186792b>] bit_wait_io+0x1b/0x70
Feb 10 13:58:34 lapio kernel: [9681928.590550] [<ffffffff818674bf>] __wait_on_bit+0x5f/0x90
Feb 10 13:58:34 lapio kernel: [9681928.590555] [<ffffffff8119857b>] wait_on_page_bit+0xcb/0xf0
Feb 10 13:58:34 lapio kernel: [9681928.590561] [<ffffffff810cad70>] ? autoremove_wake_function+0x40/0x40
Feb 10 13:58:34 lapio kernel: [9681928.590565] [<ffffffff81198693>] __filemap_fdatawait_range+0xf3/0x160
Feb 10 13:58:34 lapio kernel: [9681928.590569] [<ffffffff81198714>] filemap_fdatawait_range+0x14/0x30
Feb 10 13:58:34 lapio kernel: [9681928.590573] [<ffffffff8119a63a>] filemap_write_and_wait+0x6a/0x70
Feb 10 13:58:34 lapio kernel: [9681928.590584] [<ffffffffc0512b43>] cifs_flush+0x43/0x90 [cifs]
Feb 10 13:58:34 lapio kernel: [9681928.590589] [<ffffffff81219972>] filp_close+0x32/0x80
Feb 10 13:58:34 lapio kernel: [9681928.590594] [<ffffffff8123b3f5>] put_files_struct+0x75/0xd0
Feb 10 13:58:34 lapio kernel: [9681928.590598] [<ffffffff8123b4f7>] exit_files+0x47/0x50
Feb 10 13:58:34 lapio kernel: [9681928.590603] [<ffffffff810885ae>] do_exit+0x2ae/0xb90
Feb 10 13:58:34 lapio kernel: [9681928.590607] [<ffffffff811989fb>] ? __lock_page_killable+0xbb/0xe0
Feb 10 13:58:34 lapio kernel: [9681928.590611] [<ffffffff81088f17>] do_group_exit+0x47/0xb0
Feb 10 13:58:34 lapio kernel: [9681928.590616] [<ffffffff810957e1>] get_signal+0x171/0x950
Feb 10 13:58:34 lapio kernel: [9681928.590621] [<ffffffff8102e467>] do_signal+0x37/0x6f0
Feb 10 13:58:34 lapio kernel: [9681928.590631] [<ffffffffc0513577>] ? cifs_strict_readv+0xa7/0x100 [cifs]
Feb 10 13:58:34 lapio kernel: [9681928.590636] [<ffffffff810034fc>] exit_to_usermode_loop+0x8c/0xd0
Feb 10 13:58:34 lapio kernel: [9681928.590640] [<ffffffff81003c9a>] syscall_return_slowpath+0x5a/0x60
Feb 10 13:58:34 lapio kernel: [9681928.590644] [<ffffffff8186bd18>] int_ret_from_sys_call+0x25/0xa3
Feb 10 13:58:34 lapio kernel: [9681928.590668]
Feb 10 13:58:34 lapio kernel: [9681928.590679] Restarting tasks ... done.
Feb 10 13:58:34 lapio kernel: [9681928.602642] xen:manage: do_suspend: freeze processes failed -16
I see CIFS mentioned couple times in the VM kern log, could CIFS mounts on the VM be preventing the freeze task? I havent had this issue with this particular VM before though.
https://xcp-ng.org/docs/api.html
Restarting the API
Sometimes, you might want to restart the toolstack (API and storage stack). You can do it with the following command from the host console (or directly with Xen Orchestra):
xe-toolstack-restart
TIP
Restarting XAPI won't affect any running VMs. However, all backup/export tasks will be removed.
Switched to latest channel:
Current version: 5.99.1 - XOA build: 20241004
exact same error for me as i posted before:
Oct 08 04:12:21 xoa xo-server[152702]: 2024-10-08T09:12:21.139Z xo:backups:worker INFO starting backup
Oct 08 04:12:27 xoa xo-server[152702]: 2024-10-08T09:12:27.213Z xo:xapi:vdi INFO found changed blocks {
Oct 08 04:12:27 xoa xo-server[152702]: changedBlocks: <Buffer a0 11 60 00 00 00 00 60 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ... 511950 more bytes>
Oct 08 04:12:27 xoa xo-server[152702]: }
Oct 08 04:12:41 xoa xo-server[152702]: 2024-10-08T09:12:41.036Z xo:xapi:vdi INFO found changed blocks {
Oct 08 04:12:41 xoa xo-server[152702]: changedBlocks: <Buffer 00 00 80 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ... 102353 more bytes>
Oct 08 04:12:41 xoa xo-server[152702]: }
Oct 08 04:12:43 xoa xo-server[152702]: 2024-10-08T09:12:43.021Z xo:xapi:vdi INFO OpaqueRef:0c5189b6-8ef7-4d55-80c9-0aed7b165e18 has been disconnected from dom0 {
Oct 08 04:12:43 xoa xo-server[152702]: vdiRef: 'OpaqueRef:8b3097a7-0038-475f-a45e-5d386f3e3e26',
Oct 08 04:12:43 xoa xo-server[152702]: vbdRef: 'OpaqueRef:0c5189b6-8ef7-4d55-80c9-0aed7b165e18'
Oct 08 04:12:43 xoa xo-server[152702]: }
Oct 08 04:23:43 xoa xo-server[152702]: 2024-10-08T09:23:43.779Z xo:xapi:vdi INFO OpaqueRef:3ce8200f-f3ca-4548-b1b3-beac8b19ae2b has been disconnected from dom0 {
Oct 08 04:23:43 xoa xo-server[152702]: vdiRef: 'OpaqueRef:75005d3e-4d4c-4799-ab70-ee7e7a4f7e23',
Oct 08 04:23:43 xoa xo-server[152702]: vbdRef: 'OpaqueRef:3ce8200f-f3ca-4548-b1b3-beac8b19ae2b'
Oct 08 04:23:43 xoa xo-server[152702]: }
Oct 08 04:24:59 xoa xo-server[152702]: 2024-10-08T09:24:59.120Z xo:xapi:vdi INFO OpaqueRef:98b20faa-2c5e-4b5a-9946-63b71857f200 has been disconnected from dom0 {
Oct 08 04:24:59 xoa xo-server[152702]: vdiRef: 'OpaqueRef:89cedb32-81c9-482c-b6b0-e9d0102fd198',
Oct 08 04:24:59 xoa xo-server[152702]: vbdRef: 'OpaqueRef:98b20faa-2c5e-4b5a-9946-63b71857f200'
Oct 08 04:24:59 xoa xo-server[152702]: }
Oct 08 04:24:59 xoa xo-server[152702]: 2024-10-08T09:24:59.980Z xo:xapi WARN retry {
Oct 08 04:24:59 xoa xo-server[152702]: attemptNumber: 0,
Oct 08 04:24:59 xoa xo-server[152702]: delay: 5000,
Oct 08 04:24:59 xoa xo-server[152702]: error: XapiError: VDI_IN_USE(OpaqueRef:7d08bb03-3247-4ace-98b0-4110851a893e, destroy)
Oct 08 04:24:59 xoa xo-server[152702]: at XapiError.wrap (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/_XapiError.mjs:16:12)
Oct 08 04:24:59 xoa xo-server[152702]: at default (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/_getTaskResult.mjs:13:29)
Oct 08 04:24:59 xoa xo-server[152702]: at Xapi._addRecordToCache (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1041:24)
Oct 08 04:24:59 xoa xo-server[152702]: at file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1075:14
Oct 08 04:24:59 xoa xo-server[152702]: at Array.forEach (<anonymous>)
Oct 08 04:24:59 xoa xo-server[152702]: at Xapi._processEvents (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1065:12)
Oct 08 04:24:59 xoa xo-server[152702]: at Xapi._watchEvents (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1238:14)
Oct 08 04:24:59 xoa xo-server[152702]: at process.processTicksAndRejections (node:internal/process/task_queues:95:5) {
Oct 08 04:24:59 xoa xo-server[152702]: code: 'VDI_IN_USE',
Oct 08 04:24:59 xoa xo-server[152702]: params: [ 'OpaqueRef:7d08bb03-3247-4ace-98b0-4110851a893e', 'destroy' ],
Oct 08 04:24:59 xoa xo-server[152702]: call: undefined,
Oct 08 04:24:59 xoa xo-server[152702]: url: undefined,
Oct 08 04:24:59 xoa xo-server[152702]: task: task {
Oct 08 04:24:59 xoa xo-server[152702]: uuid: '240ae893-9d16-1762-572f-ec8da918dc49',
Oct 08 04:24:59 xoa xo-server[152702]: name_label: 'Async.VDI.destroy',
Oct 08 04:24:59 xoa xo-server[152702]: name_description: '',
Oct 08 04:24:59 xoa xo-server[152702]: allowed_operations: [],
Oct 08 04:24:59 xoa xo-server[152702]: current_operations: {},
Oct 08 04:24:59 xoa xo-server[152702]: created: '20241008T09:24:58Z',
Oct 08 04:24:59 xoa xo-server[152702]: finished: '20241008T09:24:59Z',
Oct 08 04:24:59 xoa xo-server[152702]: status: 'failure',
Oct 08 04:24:59 xoa xo-server[152702]: resident_on: 'OpaqueRef:010eebba-be27-489f-9f87-d06c8b675f19',
Oct 08 04:24:59 xoa xo-server[152702]: progress: 1,
Oct 08 04:24:59 xoa xo-server[152702]: type: '<none/>',
Oct 08 04:24:59 xoa xo-server[152702]: result: '',
Oct 08 04:24:59 xoa xo-server[152702]: error_info: [Array],
Oct 08 04:24:59 xoa xo-server[152702]: other_config: {},
Oct 08 04:24:59 xoa xo-server[152702]: subtask_of: 'OpaqueRef:NULL',
Oct 08 04:24:59 xoa xo-server[152702]: subtasks: [],
Oct 08 04:24:59 xoa xo-server[152702]: backtrace: '(((process xapi)(filename ocaml/xapi/message_forwarding.ml)(line 4711))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename ocaml/xapi/rbac.ml)(line 205))((process xapi)(filename ocaml/xapi/server_helpers.ml)(line 95)))'
Oct 08 04:24:59 xoa xo-server[152702]: }
Oct 08 04:24:59 xoa xo-server[152702]: },
Oct 08 04:24:59 xoa xo-server[152702]: fn: 'destroy',
Oct 08 04:24:59 xoa xo-server[152702]: arguments: [ 'OpaqueRef:7d08bb03-3247-4ace-98b0-4110851a893e' ],
Oct 08 04:24:59 xoa xo-server[152702]: pool: {
Oct 08 04:24:59 xoa xo-server[152702]: uuid: 'fe688bb2-b9ac-db7b-737a-cc457195f095',
Oct 08 04:24:59 xoa xo-server[152702]: name_label: 'Private Pool'
Oct 08 04:24:59 xoa xo-server[152702]: }
Oct 08 04:24:59 xoa xo-server[152702]: }
.....
Oct 08 04:30:12 xoa xo-server[152702]: 2024-10-08T09:30:12.650Z xo:xapi:vdi WARN Couldn't disconnect OpaqueRef:7d08bb03-3247-4ace-98b0-4110851a893e from dom0 {
Oct 08 04:30:12 xoa xo-server[152702]: vdiRef: 'OpaqueRef:7d08bb03-3247-4ace-98b0-4110851a893e',
Oct 08 04:30:12 xoa xo-server[152702]: vbdRef: 'OpaqueRef:872dd8e6-031c-4f12-adb5-c7bb61f9dc0f',
Oct 08 04:30:12 xoa xo-server[152702]: err: XapiError: OPERATION_NOT_ALLOWED(VBD '8817456d-774e-d82f-e63a-32a393491a0e' still attached to '9eebab44-b26e-4b6a-96e5-54734c459929')
Oct 08 04:30:12 xoa xo-server[152702]: at XapiError.wrap (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/_XapiError.mjs:16:12)
Oct 08 04:30:12 xoa xo-server[152702]: at file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/transports/json-rpc.mjs:38:21
Oct 08 04:30:12 xoa xo-server[152702]: at process.processTicksAndRejections (node:internal/process/task_queues:95:5) {
Oct 08 04:30:12 xoa xo-server[152702]: code: 'OPERATION_NOT_ALLOWED',
Oct 08 04:30:12 xoa xo-server[152702]: params: [
Oct 08 04:30:12 xoa xo-server[152702]: "VBD '8817456d-774e-d82f-e63a-32a393491a0e' still attached to '9eebab44-b26e-4b6a-96e5-54734c459929'"
Oct 08 04:30:12 xoa xo-server[152702]: ],
Oct 08 04:30:12 xoa xo-server[152702]: call: { method: 'VBD.destroy', params: [Array] },
Oct 08 04:30:12 xoa xo-server[152702]: url: undefined,
Oct 08 04:30:12 xoa xo-server[152702]: task: undefined
Oct 08 04:30:12 xoa xo-server[152702]: }
Oct 08 04:30:12 xoa xo-server[152702]: }
Oct 08 04:30:17 xoa xo-server[152702]: 2024-10-08T09:30:17.708Z xo:xapi:vm WARN VM_destroy: failed to destroy VDI {
Oct 08 04:30:17 xoa xo-server[152702]: error: XapiError: VDI_IN_USE(OpaqueRef:7d08bb03-3247-4ace-98b0-4110851a893e, destroy)
Oct 08 04:30:17 xoa xo-server[152702]: at XapiError.wrap (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/_XapiError.mjs:16:12)
Oct 08 04:30:17 xoa xo-server[152702]: at default (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/_getTaskResult.mjs:13:29)
Oct 08 04:30:17 xoa xo-server[152702]: at Xapi._addRecordToCache (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1041:24)
Oct 08 04:30:17 xoa xo-server[152702]: at file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1075:14
Oct 08 04:30:17 xoa xo-server[152702]: at Array.forEach (<anonymous>)
Oct 08 04:30:17 xoa xo-server[152702]: at Xapi._processEvents (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1065:12)
Oct 08 04:30:17 xoa xo-server[152702]: at Xapi._watchEvents (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1238:14)
Oct 08 04:30:17 xoa xo-server[152702]: at process.processTicksAndRejections (node:internal/process/task_queues:95:5) {
Oct 08 04:30:17 xoa xo-server[152702]: code: 'VDI_IN_USE',
Oct 08 04:30:17 xoa xo-server[152702]: params: [ 'OpaqueRef:7d08bb03-3247-4ace-98b0-4110851a893e', 'destroy' ],
Oct 08 04:30:17 xoa xo-server[152702]: call: undefined,
Oct 08 04:30:17 xoa xo-server[152702]: url: undefined,
Oct 08 04:30:17 xoa xo-server[152702]: task: task {
Oct 08 04:30:17 xoa xo-server[152702]: uuid: '81ce7cd4-6e11-e60d-6fc2-22522ba2dd74',
Oct 08 04:30:17 xoa xo-server[152702]: name_label: 'Async.VDI.destroy',
Oct 08 04:30:17 xoa xo-server[152702]: name_description: '',
Oct 08 04:30:17 xoa xo-server[152702]: allowed_operations: [],
Oct 08 04:30:17 xoa xo-server[152702]: current_operations: {},
Oct 08 04:30:17 xoa xo-server[152702]: created: '20241008T09:30:17Z',
Oct 08 04:30:17 xoa xo-server[152702]: finished: '20241008T09:30:17Z',
Oct 08 04:30:17 xoa xo-server[152702]: status: 'failure',
Oct 08 04:30:17 xoa xo-server[152702]: resident_on: 'OpaqueRef:010eebba-be27-489f-9f87-d06c8b675f19',
Oct 08 04:30:17 xoa xo-server[152702]: progress: 1,
Oct 08 04:30:17 xoa xo-server[152702]: type: '<none/>',
Oct 08 04:30:17 xoa xo-server[152702]: result: '',
Oct 08 04:30:17 xoa xo-server[152702]: error_info: [Array],
Oct 08 04:30:17 xoa xo-server[152702]: other_config: {},
Oct 08 04:30:17 xoa xo-server[152702]: subtask_of: 'OpaqueRef:NULL',
Oct 08 04:30:17 xoa xo-server[152702]: subtasks: [],
Oct 08 04:30:17 xoa xo-server[152702]: backtrace: '(((process xapi)(filename ocaml/xapi/message_forwarding.ml)(line 4711))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename ocaml/xapi/rbac.ml)(line 205))((process xapi)(filename ocaml/xapi/server_helpers.ml)(line 95)))'
Oct 08 04:30:17 xoa xo-server[152702]: }
Oct 08 04:30:17 xoa xo-server[152702]: },
Oct 08 04:30:17 xoa xo-server[152702]: vdiRef: 'OpaqueRef:7d08bb03-3247-4ace-98b0-4110851a893e',
Oct 08 04:30:17 xoa xo-server[152702]: vmRef: 'OpaqueRef:fac41617-6192-46b5-9d01-993da8ce039f'
Oct 08 04:30:17 xoa xo-server[152702]: }
Oct 08 04:30:18 xoa xo-server[152702]: 2024-10-08T09:30:18.041Z xo:backups:MixinBackupWriter INFO deleting unused VHD {
Oct 08 04:30:18 xoa xo-server[152702]: path: '/xo-vm-backups/d5d0334c-a7e3-b29f-51ca-1be9c211d2c1/vdis/05ce27a6-b447-4b12-b963-7e8d4978c95a/9f8da2fd-a08d-43c2-a5b1-ce6125cf52f5/20241007T135637Z.vhd'
Oct 08 04:30:18 xoa xo-server[152702]: }
Oct 08 04:30:18 xoa xo-server[152702]: 2024-10-08T09:30:18.041Z xo:backups:MixinBackupWriter INFO deleting unused VHD {
Oct 08 04:30:18 xoa xo-server[152702]: path: '/xo-vm-backups/d5d0334c-a7e3-b29f-51ca-1be9c211d2c1/vdis/05ce27a6-b447-4b12-b963-7e8d4978c95a/c6853f48-4b06-4c34-9707-b68f9054e6fc/20241007T135637Z.vhd'
Oct 08 04:30:18 xoa xo-server[152702]: }
Oct 08 04:30:18 xoa xo-server[152702]: 2024-10-08T09:30:18.148Z xo:backups:worker INFO backup has ended
Oct 08 04:30:48 xoa xo-server[152702]: 2024-10-08T09:30:48.256Z xo:backups:worker WARN worker process did not exit automatically, forcing...
Oct 08 04:30:48 xoa xo-server[152702]: 2024-10-08T09:30:48.257Z xo:backups:worker INFO process will exit {
Oct 08 04:30:48 xoa xo-server[152702]: duration: 1107117425,
Oct 08 04:30:48 xoa xo-server[152702]: exitCode: 0,
Oct 08 04:30:48 xoa xo-server[152702]: resourceUsage: {
Oct 08 04:30:48 xoa xo-server[152702]: userCPUTime: 839002829,
Oct 08 04:30:48 xoa xo-server[152702]: systemCPUTime: 176610166,
Oct 08 04:30:48 xoa xo-server[152702]: maxRSS: 89720,
Oct 08 04:30:48 xoa xo-server[152702]: sharedMemorySize: 0,
Oct 08 04:30:48 xoa xo-server[152702]: unsharedDataSize: 0,
Oct 08 04:30:48 xoa xo-server[152702]: unsharedStackSize: 0,
Oct 08 04:30:48 xoa xo-server[152702]: minorPageFault: 4860813,
Oct 08 04:30:48 xoa xo-server[152702]: majorPageFault: 0,
Oct 08 04:30:48 xoa xo-server[152702]: swappedOut: 0,
Oct 08 04:30:48 xoa xo-server[152702]: fsRead: 4224,
Oct 08 04:30:48 xoa xo-server[152702]: fsWrite: 209821368,
Oct 08 04:30:48 xoa xo-server[152702]: ipcSent: 0,
Oct 08 04:30:48 xoa xo-server[152702]: ipcReceived: 0,
Oct 08 04:30:48 xoa xo-server[152702]: signalsCount: 0,
Oct 08 04:30:48 xoa xo-server[152702]: voluntaryContextSwitches: 451688,
Oct 08 04:30:48 xoa xo-server[152702]: involuntaryContextSwitches: 480143
Oct 08 04:30:48 xoa xo-server[152702]: },
Oct 08 04:30:48 xoa xo-server[152702]: summary: { duration: '18m', cpuUsage: '92%', memoryUsage: '87.62 MiB' }
Oct 08 04:30:48 xoa xo-server[152702]: }
@olivierlambert
Same behaviour on XOA, Full backup OK, delta succeedes, but 2nd disk stuck on control domain:
Current version: 5.98.1 - XOA build: 20241004
PS. just noticed that while the delta is successfully ran, the backuplog say:
Retry the VM backup due to an error
With message:
"The value of \"offset\" is out of range. It must be >= 0 and <= 102399. Received 102400"
2nd attempt ran successfully, while leaving the vdi orphaned, on both XO and XOA.
Oct 07 08:44:39 xoa xo-server[4796]: 2024-10-07T13:44:39.083Z xo:backups:worker INFO starting backup
Oct 07 08:54:50 xoa xo-server[4796]: 2024-10-07T13:54:50.199Z xo:xapi:vdi INFO OpaqueRef:841ceab7-91b7-48a4-a85c-f4505c1614bd has been disconnected from dom0 {
Oct 07 08:54:50 xoa xo-server[4796]: vdiRef: 'OpaqueRef:01c25ee1-a119-4f1d-bfdc-f294a83fee0b',
Oct 07 08:54:50 xoa xo-server[4796]: vbdRef: 'OpaqueRef:841ceab7-91b7-48a4-a85c-f4505c1614bd'
Oct 07 08:54:50 xoa xo-server[4796]: }
Oct 07 08:54:58 xoa xo-server[4796]: 2024-10-07T13:54:58.283Z xo:backups:worker INFO backup has ended
Oct 07 08:55:09 xoa xo-server[4796]: 2024-10-07T13:55:09.682Z xo:backups:worker INFO process will exit {
Oct 07 08:55:09 xoa xo-server[4796]: duration: 630597722,
Oct 07 08:55:09 xoa xo-server[4796]: exitCode: 0,
Oct 07 08:55:09 xoa xo-server[4796]: resourceUsage: {
Oct 07 08:55:09 xoa xo-server[4796]: userCPUTime: 709081409,
Oct 07 08:55:09 xoa xo-server[4796]: systemCPUTime: 152978165,
Oct 07 08:55:09 xoa xo-server[4796]: maxRSS: 99824,
Oct 07 08:55:09 xoa xo-server[4796]: sharedMemorySize: 0,
Oct 07 08:55:09 xoa xo-server[4796]: unsharedDataSize: 0,
Oct 07 08:55:09 xoa xo-server[4796]: unsharedStackSize: 0,
Oct 07 08:55:09 xoa xo-server[4796]: minorPageFault: 1526509,
Oct 07 08:55:09 xoa xo-server[4796]: majorPageFault: 1,
Oct 07 08:55:09 xoa xo-server[4796]: swappedOut: 0,
Oct 07 08:55:09 xoa xo-server[4796]: fsRead: 4736,
Oct 07 08:55:09 xoa xo-server[4796]: fsWrite: 209788416,
Oct 07 08:55:09 xoa xo-server[4796]: ipcSent: 0,
Oct 07 08:55:09 xoa xo-server[4796]: ipcReceived: 0,
Oct 07 08:55:09 xoa xo-server[4796]: signalsCount: 0,
Oct 07 08:55:09 xoa xo-server[4796]: voluntaryContextSwitches: 397016,
Oct 07 08:55:09 xoa xo-server[4796]: involuntaryContextSwitches: 441949
Oct 07 08:55:09 xoa xo-server[4796]: },
Oct 07 08:55:09 xoa xo-server[4796]: summary: { duration: '11m', cpuUsage: '137%', memoryUsage: '97.48 MiB' }
Oct 07 08:55:09 xoa xo-server[4796]: }
Oct 07 08:55:47 xoa xo-server[6339]: 2024-10-07T13:55:47.446Z xo:backups:worker INFO starting backup
Oct 07 08:55:53 xoa xo-server[6339]: 2024-10-07T13:55:53.223Z xo:xapi:vdi INFO found changed blocks {
Oct 07 08:55:53 xoa xo-server[6339]: changedBlocks: <Buffer 80 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ... 511950 more bytes>
Oct 07 08:55:53 xoa xo-server[6339]: }
Oct 07 08:56:11 xoa xo-server[6339]: 2024-10-07T13:56:11.907Z xo:xapi:vdi INFO found changed blocks {
Oct 07 08:56:11 xoa xo-server[6339]: changedBlocks: <Buffer 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ... 102353 more bytes>
Oct 07 08:56:11 xoa xo-server[6339]: }
Oct 07 08:56:14 xoa xo-server[6339]: 2024-10-07T13:56:14.369Z xo:xapi:vdi INFO OpaqueRef:18023c91-93c8-489b-85e4-f2bda338258c has been disconnected from dom0 {
Oct 07 08:56:14 xoa xo-server[6339]: vdiRef: 'OpaqueRef:b438dd7b-d27c-4184-99e5-fddfb44901b0',
Oct 07 08:56:14 xoa xo-server[6339]: vbdRef: 'OpaqueRef:18023c91-93c8-489b-85e4-f2bda338258c'
Oct 07 08:56:14 xoa xo-server[6339]: }
Oct 07 09:07:42 xoa xo-server[6339]: 2024-10-07T14:07:42.981Z xo:xapi WARN retry {
Oct 07 09:07:42 xoa xo-server[6339]: attemptNumber: 0,
Oct 07 09:07:42 xoa xo-server[6339]: delay: 5000,
Oct 07 09:07:42 xoa xo-server[6339]: error: XapiError: VDI_IN_USE(OpaqueRef:69e0e65f-94ee-468a-b919-2b22d3db670a, destroy)
Oct 07 09:07:42 xoa xo-server[6339]: at XapiError.wrap (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/_XapiError.mjs:16:12)
Oct 07 09:07:42 xoa xo-server[6339]: at default (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/_getTaskResult.mjs:13:29)
Oct 07 09:07:42 xoa xo-server[6339]: at Xapi._addRecordToCache (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1076:24)
Oct 07 09:07:42 xoa xo-server[6339]: at file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1110:14
Oct 07 09:07:42 xoa xo-server[6339]: at Array.forEach (<anonymous>)
Oct 07 09:07:42 xoa xo-server[6339]: at Xapi._processEvents (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1100:12)
Oct 07 09:07:42 xoa xo-server[6339]: at Xapi._watchEvents (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1273:14)
Oct 07 09:07:42 xoa xo-server[6339]: at process.processTicksAndRejections (node:internal/process/task_queues:95:5) {
Oct 07 09:07:42 xoa xo-server[6339]: code: 'VDI_IN_USE',
Oct 07 09:07:42 xoa xo-server[6339]: params: [ 'OpaqueRef:69e0e65f-94ee-468a-b919-2b22d3db670a', 'destroy' ],
Oct 07 09:07:42 xoa xo-server[6339]: call: undefined,
Oct 07 09:07:42 xoa xo-server[6339]: url: undefined,
Oct 07 09:07:42 xoa xo-server[6339]: task: task {
Oct 07 09:07:42 xoa xo-server[6339]: uuid: '30d84ccc-fe22-fd16-6697-4615df096ee6',
Oct 07 09:07:42 xoa xo-server[6339]: name_label: 'Async.VDI.destroy',
Oct 07 09:07:42 xoa xo-server[6339]: name_description: '',
Oct 07 09:07:42 xoa xo-server[6339]: allowed_operations: [],
Oct 07 09:07:42 xoa xo-server[6339]: current_operations: {},
Oct 07 09:07:42 xoa xo-server[6339]: created: '20241007T14:07:33Z',
Oct 07 09:07:42 xoa xo-server[6339]: finished: '20241007T14:07:42Z',
Oct 07 09:07:42 xoa xo-server[6339]: status: 'failure',
Oct 07 09:07:42 xoa xo-server[6339]: resident_on: 'OpaqueRef:010eebba-be27-489f-9f87-d06c8b675f19',
Oct 07 09:07:42 xoa xo-server[6339]: progress: 1,
Oct 07 09:07:42 xoa xo-server[6339]: type: '<none/>',
Oct 07 09:07:42 xoa xo-server[6339]: result: '',
Oct 07 09:07:42 xoa xo-server[6339]: error_info: [Array],
Oct 07 09:07:42 xoa xo-server[6339]: other_config: {},
Oct 07 09:07:42 xoa xo-server[6339]: subtask_of: 'OpaqueRef:NULL',
Oct 07 09:07:42 xoa xo-server[6339]: subtasks: [],
Oct 07 09:07:42 xoa xo-server[6339]: backtrace: '(((process xapi)(filename ocaml/xapi/message_forwarding.ml)(line 4711))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename ocaml/xapi/rbac.ml)(line 205))((process xapi)(filename o>
Oct 07 09:07:42 xoa xo-server[6339]: }
Oct 07 09:07:42 xoa xo-server[6339]: },
Oct 07 09:07:42 xoa xo-server[6339]: fn: 'destroy',
Oct 07 09:07:42 xoa xo-server[6339]: arguments: [ 'OpaqueRef:69e0e65f-94ee-468a-b919-2b22d3db670a' ],
Oct 07 09:07:42 xoa xo-server[6339]: pool: {
Oct 07 09:07:42 xoa xo-server[6339]: uuid: 'fe688bb2-b9ac-db7b-737a-cc457195f095',
Oct 07 09:07:42 xoa xo-server[6339]: name_label: 'SGO Pool'
Oct 07 09:07:42 xoa xo-server[6339]: }
Oct 07 09:07:42 xoa xo-server[6339]: }
Oct 07 09:08:28 xoa xo-server[6339]: 2024-10-07T14:08:28.493Z xo:xapi:vm WARN VM_destroy: failed to destroy VDI {
Oct 07 09:08:28 xoa xo-server[6339]: error: XapiError: VDI_IN_USE(OpaqueRef:69e0e65f-94ee-468a-b919-2b22d3db670a, destroy)
Oct 07 09:08:28 xoa xo-server[6339]: at XapiError.wrap (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/_XapiError.mjs:16:12)
Oct 07 09:08:28 xoa xo-server[6339]: at default (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/_getTaskResult.mjs:13:29)
Oct 07 09:08:28 xoa xo-server[6339]: at Xapi._addRecordToCache (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1076:24)
Oct 07 09:08:28 xoa xo-server[6339]: at file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1110:14
Oct 07 09:08:28 xoa xo-server[6339]: at Array.forEach (<anonymous>)
Oct 07 09:08:28 xoa xo-server[6339]: at Xapi._processEvents (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1100:12)
Oct 07 09:08:28 xoa xo-server[6339]: at Xapi._watchEvents (file:///usr/local/lib/node_modules/xo-server/node_modules/xen-api/index.mjs:1273:14)
Oct 07 09:08:28 xoa xo-server[6339]: at process.processTicksAndRejections (node:internal/process/task_queues:95:5) {
Oct 07 09:08:28 xoa xo-server[6339]: code: 'VDI_IN_USE',
Oct 07 09:08:28 xoa xo-server[6339]: params: [ 'OpaqueRef:69e0e65f-94ee-468a-b919-2b22d3db670a', 'destroy' ],
Oct 07 09:08:28 xoa xo-server[6339]: call: undefined,
Oct 07 09:08:28 xoa xo-server[6339]: url: undefined,
Oct 07 09:08:28 xoa xo-server[6339]: task: task {
Oct 07 09:08:28 xoa xo-server[6339]: uuid: '76ceadda-070d-3e2e-282b-76c58abd13b1',
Oct 07 09:08:28 xoa xo-server[6339]: name_label: 'Async.VDI.destroy',
Oct 07 09:08:28 xoa xo-server[6339]: name_description: '',
Oct 07 09:08:28 xoa xo-server[6339]: allowed_operations: [],
Oct 07 09:08:28 xoa xo-server[6339]: current_operations: {},
Oct 07 09:08:28 xoa xo-server[6339]: created: '20241007T14:08:28Z',
Oct 07 09:08:28 xoa xo-server[6339]: finished: '20241007T14:08:28Z',
Oct 07 09:08:28 xoa xo-server[6339]: status: 'failure',
Oct 07 09:08:28 xoa xo-server[6339]: resident_on: 'OpaqueRef:010eebba-be27-489f-9f87-d06c8b675f19',
Oct 07 09:08:28 xoa xo-server[6339]: progress: 1,
Oct 07 09:08:28 xoa xo-server[6339]: type: '<none/>',
Oct 07 09:08:28 xoa xo-server[6339]: result: '',
Oct 07 09:08:28 xoa xo-server[6339]: error_info: [Array],
Oct 07 09:08:28 xoa xo-server[6339]: other_config: {},
Oct 07 09:08:28 xoa xo-server[6339]: subtask_of: 'OpaqueRef:NULL',
Oct 07 09:08:28 xoa xo-server[6339]: subtasks: [],
Oct 07 09:08:28 xoa xo-server[6339]: backtrace: '(((process xapi)(filename ocaml/xapi/message_forwarding.ml)(line 4711))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename ocaml/xapi/rbac.ml)(line 205))((process xapi)(filename o>
Oct 07 09:08:28 xoa xo-server[6339]: }
Oct 07 09:08:28 xoa xo-server[6339]: },
Oct 07 09:08:28 xoa xo-server[6339]: vdiRef: 'OpaqueRef:69e0e65f-94ee-468a-b919-2b22d3db670a',
Oct 07 09:08:28 xoa xo-server[6339]: vmRef: 'OpaqueRef:c246a561-2409-4e72-900c-b2d259707e22'
Oct 07 09:08:28 xoa xo-server[6339]: }
Oct 07 09:08:28 xoa xo-server[6339]: 2024-10-07T14:08:28.677Z xo:backups:MixinBackupWriter INFO deleting unused VHD {
Oct 07 09:08:28 xoa xo-server[6339]: path: '/xo-vm-backups/d5d0334c-a7e3-b29f-51ca-1be9c211d2c1/vdis/05ce27a6-b447-4b12-b963-7e8d4978c95a/c6853f48-4b06-4c34-9707-b68f9054e6fc/20241007T134447Z.vhd'
Oct 07 09:08:28 xoa xo-server[6339]: }
Oct 07 09:08:28 xoa xo-server[6339]: 2024-10-07T14:08:28.678Z xo:backups:MixinBackupWriter INFO deleting unused VHD {
Oct 07 09:08:28 xoa xo-server[6339]: path: '/xo-vm-backups/d5d0334c-a7e3-b29f-51ca-1be9c211d2c1/vdis/05ce27a6-b447-4b12-b963-7e8d4978c95a/9f8da2fd-a08d-43c2-a5b1-ce6125cf52f5/20241007T134447Z.vhd'
Oct 07 09:08:28 xoa xo-server[6339]: }
Oct 07 09:08:28 xoa xo-server[6339]: 2024-10-07T14:08:28.788Z xo:backups:worker INFO backup has ended
Hi,
I have a problem with a single VM running Almalinux 9. The VM has 2 disks, 50GB and 250GB disk.
Every time i do Delta backup the backup does succeed, but the larger disk is ALWAYS stuck on Control Domain and i have to manually "forget" it, or next backup will fail, which is annoying.
This does not occur on full backup, only delta.
I removed all previous snapshots and backup and retried and same thing is happening.
Using XO with latest patches: commit a5967.
This issue started occurring straight after the CBT feature was added.
Using NBD = 1
Here is XO LOG for the backup:
Omitted attempts 2-7.... since they don't fit here
Oct 7 11:33:43 xo-ce xo-server[11768]: 2024-10-07T11:33:43.770Z xo:backups:worker INFO starting backup
Oct 7 11:33:49 xo-ce xo-server[11768]: 2024-10-07T11:33:49.717Z xo:xapi:vdi INFO found changed blocks {
Oct 7 11:33:49 xo-ce xo-server[11768]: changedBlocks: <Buffer 80 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ... 511950 more bytes>
Oct 7 11:33:49 xo-ce xo-server[11768]: }
Oct 7 11:34:09 xo-ce xo-server[11768]: 2024-10-07T11:34:09.098Z xo:xapi:vdi INFO found changed blocks {
Oct 7 11:34:09 xo-ce xo-server[11768]: changedBlocks: <Buffer 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ... 102353 more bytes>
Oct 7 11:34:09 xo-ce xo-server[11768]: }
Oct 7 11:34:11 xo-ce xo-server[11768]: 2024-10-07T11:34:11.070Z xo:xapi:vdi INFO OpaqueRef:c1f4b3cf-996d-45ba-80ae-59ef7c9cb8ef has been disconnected from dom0 {
Oct 7 11:34:11 xo-ce xo-server[11768]: vdiRef: 'OpaqueRef:2fd1d459-46b5-473a-9549-a3e0b401209f',
Oct 7 11:34:11 xo-ce xo-server[11768]: vbdRef: 'OpaqueRef:c1f4b3cf-996d-45ba-80ae-59ef7c9cb8ef'
Oct 7 11:34:11 xo-ce xo-server[11768]: }
Oct 7 11:40:47 xo-ce xo-server[11768]: 2024-10-07T11:40:47.534Z xo:xapi:vdi INFO OpaqueRef:7e94c0da-be46-45f8-9a6b-7e9eb04928b1 has been disconnected from dom0 {
Oct 7 11:40:47 xo-ce xo-server[11768]: vdiRef: 'OpaqueRef:f8be3361-0c5c-48b4-bf51-baee4d02cb20',
Oct 7 11:40:47 xo-ce xo-server[11768]: vbdRef: 'OpaqueRef:7e94c0da-be46-45f8-9a6b-7e9eb04928b1'
Oct 7 11:40:47 xo-ce xo-server[11768]: }
Oct 7 11:41:59 xo-ce xo-server[11768]: 2024-10-07T11:41:59.303Z xo:xapi:vdi INFO OpaqueRef:2e1231be-f65b-4fa4-bb8d-3871ddabfa87 has been disconnected from dom0 {
Oct 7 11:41:59 xo-ce xo-server[11768]: vdiRef: 'OpaqueRef:cb2af589-ae7e-4b5c-b657-0c2636b3677f',
Oct 7 11:41:59 xo-ce xo-server[11768]: vbdRef: 'OpaqueRef:2e1231be-f65b-4fa4-bb8d-3871ddabfa87'
Oct 7 11:41:59 xo-ce xo-server[11768]: }
Oct 7 11:42:11 xo-ce xo-server[11768]: 2024-10-07T11:42:11.742Z xo:xapi WARN retry {
Oct 7 11:42:11 xo-ce xo-server[11768]: attemptNumber: 0,
Oct 7 11:42:11 xo-ce xo-server[11768]: delay: 5000,
Oct 7 11:42:11 xo-ce xo-server[11768]: error: XapiError: VDI_IN_USE(OpaqueRef:8673586d-a9fc-4651-883a-ad333c13dc92, destroy)
Oct 7 11:42:11 xo-ce xo-server[11768]: at XapiError.wrap (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/_XapiError.mjs:16:12)
Oct 7 11:42:11 xo-ce xo-server[11768]: at default (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/_getTaskResult.mjs:13:29)
Oct 7 11:42:11 xo-ce xo-server[11768]: at Xapi._addRecordToCache (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1041:24)
Oct 7 11:42:11 xo-ce xo-server[11768]: at file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1075:14
Oct 7 11:42:11 xo-ce xo-server[11768]: at Array.forEach (<anonymous>)
Oct 7 11:42:11 xo-ce xo-server[11768]: at Xapi._processEvents (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1065:12)
Oct 7 11:42:11 xo-ce xo-server[11768]: at Xapi._watchEvents (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1238:14)
Oct 7 11:42:11 xo-ce xo-server[11768]: at process.processTicksAndRejections (node:internal/process/task_queues:95:5) {
Oct 7 11:42:11 xo-ce xo-server[11768]: code: 'VDI_IN_USE',
Oct 7 11:42:11 xo-ce xo-server[11768]: params: [ 'OpaqueRef:8673586d-a9fc-4651-883a-ad333c13dc92', 'destroy' ],
Oct 7 11:42:11 xo-ce xo-server[11768]: call: undefined,
Oct 7 11:42:11 xo-ce xo-server[11768]: url: undefined,
Oct 7 11:42:11 xo-ce xo-server[11768]: task: task {
Oct 7 11:42:11 xo-ce xo-server[11768]: uuid: '7bb5adaa-5b49-2227-3ede-759420836f59',
Oct 7 11:42:11 xo-ce xo-server[11768]: name_label: 'Async.VDI.destroy',
Oct 7 11:42:11 xo-ce xo-server[11768]: name_description: '',
Oct 7 11:42:11 xo-ce xo-server[11768]: allowed_operations: [],
Oct 7 11:42:11 xo-ce xo-server[11768]: current_operations: {},
Oct 7 11:42:11 xo-ce xo-server[11768]: created: '20241007T11:41:59Z',
Oct 7 11:42:11 xo-ce xo-server[11768]: finished: '20241007T11:42:11Z',
Oct 7 11:42:11 xo-ce xo-server[11768]: status: 'failure',
Oct 7 11:42:11 xo-ce xo-server[11768]: resident_on: 'OpaqueRef:010eebba-be27-489f-9f87-d06c8b675f19',
Oct 7 11:42:11 xo-ce xo-server[11768]: progress: 1,
Oct 7 11:42:11 xo-ce xo-server[11768]: type: '<none/>',
Oct 7 11:42:11 xo-ce xo-server[11768]: result: '',
Oct 7 11:42:11 xo-ce xo-server[11768]: error_info: [Array],
Oct 7 11:42:11 xo-ce xo-server[11768]: other_config: {},
Oct 7 11:42:11 xo-ce xo-server[11768]: subtask_of: 'OpaqueRef:NULL',
Oct 7 11:42:11 xo-ce xo-server[11768]: subtasks: [],
Oct 7 11:42:11 xo-ce xo-server[11768]: backtrace: '(((process xapi)(filename ocaml/xapi/message_forwarding.ml)(line 4711))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename ocaml/xapi/rbac.ml)(line 205))((process xapi)(filename ocaml/xapi/server_helpers.ml)(line 95)))'
Oct 7 11:42:11 xo-ce xo-server[11768]: }
Oct 7 11:42:11 xo-ce xo-server[11768]: },
Oct 7 11:42:11 xo-ce xo-server[11768]: fn: 'destroy',
Oct 7 11:42:11 xo-ce xo-server[11768]: arguments: [ 'OpaqueRef:8673586d-a9fc-4651-883a-ad333c13dc92' ],
Oct 7 11:42:11 xo-ce xo-server[11768]: pool: {
Oct 7 11:42:11 xo-ce xo-server[11768]: uuid: 'fe688bb2-b9ac-db7b-737a-cc457195f095',
Oct 7 11:42:11 xo-ce xo-server[11768]: name_label: 'SGO Pool'
Oct 7 11:42:11 xo-ce xo-server[11768]: }
Oct 7 11:42:11 xo-ce xo-server[11768]: }
Oct 7 11:42:42 xo-ce xo-server[11768]: 2024-10-07T11:42:42.061Z xo:xapi:vdi WARN Couldn't disconnect OpaqueRef:8673586d-a9fc-4651-883a-ad333c13dc92 from dom0 {
Oct 7 11:42:42 xo-ce xo-server[11768]: vdiRef: 'OpaqueRef:8673586d-a9fc-4651-883a-ad333c13dc92',
Oct 7 11:42:42 xo-ce xo-server[11768]: vbdRef: 'OpaqueRef:2b0c22e7-2794-471a-bd85-145cf302f8e4',
Oct 7 11:42:42 xo-ce xo-server[11768]: err: XapiError: OPERATION_NOT_ALLOWED(VBD '0da0be96-45ef-684e-b956-7b5463cd9b09' still attached to 'f3fd6bd3-a622-4838-a7b6-e6e5e2cc6d04')
Oct 7 11:42:42 xo-ce xo-server[11768]: at XapiError.wrap (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/_XapiError.mjs:16:12)
Oct 7 11:42:42 xo-ce xo-server[11768]: at file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/transports/json-rpc.mjs:38:21
Oct 7 11:42:42 xo-ce xo-server[11768]: at process.processTicksAndRejections (node:internal/process/task_queues:95:5) {
Oct 7 11:42:42 xo-ce xo-server[11768]: code: 'OPERATION_NOT_ALLOWED',
Oct 7 11:42:42 xo-ce xo-server[11768]: params: [
Oct 7 11:42:42 xo-ce xo-server[11768]: "VBD '0da0be96-45ef-684e-b956-7b5463cd9b09' still attached to 'f3fd6bd3-a622-4838-a7b6-e6e5e2cc6d04'"
Oct 7 11:42:42 xo-ce xo-server[11768]: ],
Oct 7 11:42:42 xo-ce xo-server[11768]: call: { method: 'VBD.destroy', params: [Array] },
Oct 7 11:42:42 xo-ce xo-server[11768]: url: undefined,
Oct 7 11:42:42 xo-ce xo-server[11768]: task: undefined
Oct 7 11:42:42 xo-ce xo-server[11768]: }
Oct 7 11:42:42 xo-ce xo-server[11768]: }
Oct 7 11:46:54 xo-ce xo-server[11768]: 2024-10-07T11:46:54.842Z xo:xapi WARN retry {
Oct 7 11:46:54 xo-ce xo-server[11768]: attemptNumber: 8,
Oct 7 11:46:54 xo-ce xo-server[11768]: delay: 5000,
Oct 7 11:46:54 xo-ce xo-server[11768]: error: XapiError: VDI_IN_USE(OpaqueRef:8673586d-a9fc-4651-883a-ad333c13dc92, destroy)
Oct 7 11:46:54 xo-ce xo-server[11768]: at XapiError.wrap (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/_XapiError.mjs:16:12)
Oct 7 11:46:54 xo-ce xo-server[11768]: at default (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/_getTaskResult.mjs:13:29)
Oct 7 11:46:54 xo-ce xo-server[11768]: at Xapi._addRecordToCache (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1041:24)
Oct 7 11:46:54 xo-ce xo-server[11768]: at file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1075:14
Oct 7 11:46:54 xo-ce xo-server[11768]: at Array.forEach (<anonymous>)
Oct 7 11:46:54 xo-ce xo-server[11768]: at Xapi._processEvents (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1065:12)
Oct 7 11:46:54 xo-ce xo-server[11768]: at Xapi._watchEvents (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1238:14)
Oct 7 11:46:54 xo-ce xo-server[11768]: at process.processTicksAndRejections (node:internal/process/task_queues:95:5) {
Oct 7 11:46:54 xo-ce xo-server[11768]: code: 'VDI_IN_USE',
Oct 7 11:46:54 xo-ce xo-server[11768]: params: [ 'OpaqueRef:8673586d-a9fc-4651-883a-ad333c13dc92', 'destroy' ],
Oct 7 11:46:54 xo-ce xo-server[11768]: call: undefined,
Oct 7 11:46:54 xo-ce xo-server[11768]: url: undefined,
Oct 7 11:46:54 xo-ce xo-server[11768]: task: task {
Oct 7 11:46:54 xo-ce xo-server[11768]: uuid: '70afe347-815d-806b-0eed-dd6239adcf0e',
Oct 7 11:46:54 xo-ce xo-server[11768]: name_label: 'Async.VDI.destroy',
Oct 7 11:46:54 xo-ce xo-server[11768]: name_description: '',
Oct 7 11:46:54 xo-ce xo-server[11768]: allowed_operations: [],
Oct 7 11:46:54 xo-ce xo-server[11768]: current_operations: {},
Oct 7 11:46:54 xo-ce xo-server[11768]: created: '20241007T11:46:54Z',
Oct 7 11:46:54 xo-ce xo-server[11768]: finished: '20241007T11:46:54Z',
Oct 7 11:46:54 xo-ce xo-server[11768]: status: 'failure',
Oct 7 11:46:54 xo-ce xo-server[11768]: resident_on: 'OpaqueRef:010eebba-be27-489f-9f87-d06c8b675f19',
Oct 7 11:46:54 xo-ce xo-server[11768]: progress: 1,
Oct 7 11:46:54 xo-ce xo-server[11768]: type: '<none/>',
Oct 7 11:46:54 xo-ce xo-server[11768]: result: '',
Oct 7 11:46:54 xo-ce xo-server[11768]: error_info: [Array],
Oct 7 11:46:54 xo-ce xo-server[11768]: other_config: {},
Oct 7 11:46:54 xo-ce xo-server[11768]: subtask_of: 'OpaqueRef:NULL',
Oct 7 11:46:54 xo-ce xo-server[11768]: subtasks: [],
Oct 7 11:46:54 xo-ce xo-server[11768]: backtrace: '(((process xapi)(filename ocaml/xapi/message_forwarding.ml)(line 4711))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename ocaml/xapi/rbac.ml)(line 205))((process xapi)(filename ocaml/xapi/server_helpers.ml)(line 95)))'
Oct 7 11:46:54 xo-ce xo-server[11768]: }
Oct 7 11:46:54 xo-ce xo-server[11768]: },
Oct 7 11:46:54 xo-ce xo-server[11768]: fn: 'destroy',
Oct 7 11:46:54 xo-ce xo-server[11768]: arguments: [ 'OpaqueRef:8673586d-a9fc-4651-883a-ad333c13dc92' ],
Oct 7 11:46:54 xo-ce xo-server[11768]: pool: {
Oct 7 11:46:54 xo-ce xo-server[11768]: uuid: 'fe688bb2-b9ac-db7b-737a-cc457195f095',
Oct 7 11:46:54 xo-ce xo-server[11768]: name_label: 'SGO Pool'
Oct 7 11:46:54 xo-ce xo-server[11768]: }
Oct 7 11:46:54 xo-ce xo-server[11768]: }
Oct 7 11:47:25 xo-ce xo-server[11768]: 2024-10-07T11:47:25.194Z xo:xapi:vdi WARN Couldn't disconnect OpaqueRef:8673586d-a9fc-4651-883a-ad333c13dc92 from dom0 {
Oct 7 11:47:25 xo-ce xo-server[11768]: vdiRef: 'OpaqueRef:8673586d-a9fc-4651-883a-ad333c13dc92',
Oct 7 11:47:25 xo-ce xo-server[11768]: vbdRef: 'OpaqueRef:2b0c22e7-2794-471a-bd85-145cf302f8e4',
Oct 7 11:47:25 xo-ce xo-server[11768]: err: XapiError: OPERATION_NOT_ALLOWED(VBD '0da0be96-45ef-684e-b956-7b5463cd9b09' still attached to 'f3fd6bd3-a622-4838-a7b6-e6e5e2cc6d04')
Oct 7 11:47:25 xo-ce xo-server[11768]: at XapiError.wrap (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/_XapiError.mjs:16:12)
Oct 7 11:47:25 xo-ce xo-server[11768]: at file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/transports/json-rpc.mjs:38:21
Oct 7 11:47:25 xo-ce xo-server[11768]: at process.processTicksAndRejections (node:internal/process/task_queues:95:5) {
Oct 7 11:47:25 xo-ce xo-server[11768]: code: 'OPERATION_NOT_ALLOWED',
Oct 7 11:47:25 xo-ce xo-server[11768]: params: [
Oct 7 11:47:25 xo-ce xo-server[11768]: "VBD '0da0be96-45ef-684e-b956-7b5463cd9b09' still attached to 'f3fd6bd3-a622-4838-a7b6-e6e5e2cc6d04'"
Oct 7 11:47:25 xo-ce xo-server[11768]: ],
Oct 7 11:47:25 xo-ce xo-server[11768]: call: { method: 'VBD.destroy', params: [Array] },
Oct 7 11:47:25 xo-ce xo-server[11768]: url: undefined,
Oct 7 11:47:25 xo-ce xo-server[11768]: task: undefined
Oct 7 11:47:25 xo-ce xo-server[11768]: }
Oct 7 11:47:25 xo-ce xo-server[11768]: }
Oct 7 11:47:30 xo-ce xo-server[11768]: 2024-10-07T11:47:30.323Z xo:xapi:vm WARN VM_destroy: failed to destroy VDI {
Oct 7 11:47:30 xo-ce xo-server[11768]: error: XapiError: VDI_IN_USE(OpaqueRef:8673586d-a9fc-4651-883a-ad333c13dc92, destroy)
Oct 7 11:47:30 xo-ce xo-server[11768]: at XapiError.wrap (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/_XapiError.mjs:16:12)
Oct 7 11:47:30 xo-ce xo-server[11768]: at default (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/_getTaskResult.mjs:13:29)
Oct 7 11:47:30 xo-ce xo-server[11768]: at Xapi._addRecordToCache (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1041:24)
Oct 7 11:47:30 xo-ce xo-server[11768]: at file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1075:14
Oct 7 11:47:30 xo-ce xo-server[11768]: at Array.forEach (<anonymous>)
Oct 7 11:47:30 xo-ce xo-server[11768]: at Xapi._processEvents (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1065:12)
Oct 7 11:47:30 xo-ce xo-server[11768]: at Xapi._watchEvents (file:///opt/xo/xo-builds/xen-orchestra-202410070635/packages/xen-api/index.mjs:1238:14)
Oct 7 11:47:30 xo-ce xo-server[11768]: at process.processTicksAndRejections (node:internal/process/task_queues:95:5) {
Oct 7 11:47:30 xo-ce xo-server[11768]: code: 'VDI_IN_USE',
Oct 7 11:47:30 xo-ce xo-server[11768]: params: [ 'OpaqueRef:8673586d-a9fc-4651-883a-ad333c13dc92', 'destroy' ],
Oct 7 11:47:30 xo-ce xo-server[11768]: call: undefined,
Oct 7 11:47:30 xo-ce xo-server[11768]: url: undefined,
Oct 7 11:47:30 xo-ce xo-server[11768]: task: task {
Oct 7 11:47:30 xo-ce xo-server[11768]: uuid: '5c688ec4-5523-c1f1-89dc-a881abdbaee1',
Oct 7 11:47:30 xo-ce xo-server[11768]: name_label: 'Async.VDI.destroy',
Oct 7 11:47:30 xo-ce xo-server[11768]: name_description: '',
Oct 7 11:47:30 xo-ce xo-server[11768]: allowed_operations: [],
Oct 7 11:47:30 xo-ce xo-server[11768]: current_operations: {},
Oct 7 11:47:30 xo-ce xo-server[11768]: created: '20241007T11:47:30Z',
Oct 7 11:47:30 xo-ce xo-server[11768]: finished: '20241007T11:47:30Z',
Oct 7 11:47:30 xo-ce xo-server[11768]: status: 'failure',
Oct 7 11:47:30 xo-ce xo-server[11768]: resident_on: 'OpaqueRef:010eebba-be27-489f-9f87-d06c8b675f19',
Oct 7 11:47:30 xo-ce xo-server[11768]: progress: 1,
Oct 7 11:47:30 xo-ce xo-server[11768]: type: '<none/>',
Oct 7 11:47:30 xo-ce xo-server[11768]: result: '',
Oct 7 11:47:30 xo-ce xo-server[11768]: error_info: [Array],
Oct 7 11:47:30 xo-ce xo-server[11768]: other_config: {},
Oct 7 11:47:30 xo-ce xo-server[11768]: subtask_of: 'OpaqueRef:NULL',
Oct 7 11:47:30 xo-ce xo-server[11768]: subtasks: [],
Oct 7 11:47:30 xo-ce xo-server[11768]: backtrace: '(((process xapi)(filename ocaml/xapi/message_forwarding.ml)(line 4711))((process xapi)(filename lib/xapi-stdext-pervasives/pervasiveext.ml)(line 24))((process xapi)(filename ocaml/xapi/rbac.ml)(line 205))((process xapi)(filename ocaml/xapi/server_helpers.ml)(line 95)))'
Oct 7 11:47:30 xo-ce xo-server[11768]: }
Oct 7 11:47:30 xo-ce xo-server[11768]: },
Oct 7 11:47:30 xo-ce xo-server[11768]: vdiRef: 'OpaqueRef:8673586d-a9fc-4651-883a-ad333c13dc92',
Oct 7 11:47:30 xo-ce xo-server[11768]: vmRef: 'OpaqueRef:239e4aa1-b3ce-4fa0-abe4-3fb6803a990e'
Oct 7 11:47:30 xo-ce xo-server[11768]: }
Oct 7 11:47:30 xo-ce xo-server[11768]: 2024-10-07T11:47:30.474Z xo:backups:MixinBackupWriter INFO deleting unused VHD {
Oct 7 11:47:30 xo-ce xo-server[11768]: path: '/xo-vm-backups/d5d0334c-a7e3-b29f-51ca-1be9c211d2c1/vdis/5a7a5647-d2ca-4014-b87b-06c621c09a20/9f8da2fd-a08d-43c2-a5b1-ce6125cf52f5/20241007T111850Z.vhd'
Oct 7 11:47:30 xo-ce xo-server[11768]: }
Oct 7 11:47:30 xo-ce xo-server[11768]: 2024-10-07T11:47:30.475Z xo:backups:MixinBackupWriter INFO deleting unused VHD {
Oct 7 11:47:30 xo-ce xo-server[11768]: path: '/xo-vm-backups/d5d0334c-a7e3-b29f-51ca-1be9c211d2c1/vdis/5a7a5647-d2ca-4014-b87b-06c621c09a20/c6853f48-4b06-4c34-9707-b68f9054e6fc/20241007T111850Z.vhd'
Oct 7 11:47:30 xo-ce xo-server[11768]: }
Oct 7 11:47:30 xo-ce xo-server[11768]: 2024-10-07T11:47:30.568Z xo:backups:worker INFO backup has ended
Oct 7 11:48:00 xo-ce xo-server[11768]: 2024-10-07T11:48:00.630Z xo:backups:worker WARN worker process did not exit automatically, forcing...
Oct 7 11:48:00 xo-ce xo-server[11768]: 2024-10-07T11:48:00.631Z xo:backups:worker INFO process will exit {
Oct 7 11:48:00 xo-ce xo-server[11768]: duration: 856860228,
Oct 7 11:48:00 xo-ce xo-server[11768]: exitCode: 0,
Oct 7 11:48:00 xo-ce xo-server[11768]: resourceUsage: {
Oct 7 11:48:00 xo-ce xo-server[11768]: userCPUTime: 771667771,
Oct 7 11:48:00 xo-ce xo-server[11768]: systemCPUTime: 176351529,
Oct 7 11:48:00 xo-ce xo-server[11768]: maxRSS: 0,
Oct 7 11:48:00 xo-ce xo-server[11768]: sharedMemorySize: 0,
Oct 7 11:48:00 xo-ce xo-server[11768]: unsharedDataSize: 0,
Oct 7 11:48:00 xo-ce xo-server[11768]: unsharedStackSize: 0,
Oct 7 11:48:00 xo-ce xo-server[11768]: minorPageFault: 3322393,
Oct 7 11:48:00 xo-ce xo-server[11768]: majorPageFault: 0,
Oct 7 11:48:00 xo-ce xo-server[11768]: swappedOut: 0,
Oct 7 11:48:00 xo-ce xo-server[11768]: fsRead: 2416,
Oct 7 11:48:00 xo-ce xo-server[11768]: fsWrite: 209789336,
Oct 7 11:48:00 xo-ce xo-server[11768]: ipcSent: 0,
Oct 7 11:48:00 xo-ce xo-server[11768]: ipcReceived: 0,
Oct 7 11:48:00 xo-ce xo-server[11768]: signalsCount: 0,
Oct 7 11:48:00 xo-ce xo-server[11768]: voluntaryContextSwitches: 527248,
Oct 7 11:48:00 xo-ce xo-server[11768]: involuntaryContextSwitches: 342752
Oct 7 11:48:00 xo-ce xo-server[11768]: },
Oct 7 11:48:00 xo-ce xo-server[11768]: summary: { duration: '14m', cpuUsage: '111%', memoryUsage: '0 B' }
Oct 7 11:48:00 xo-ce xo-server[11768]: }