Backup Jobs failing
-
I have a few vms that are failing backup. XO-CE is 2 commits behind.
Just updated to lates commit 08c29 and still failing. One is windows vm other linux vm.
{ "data": { "mode": "delta", "reportWhen": "always", "backupReportTpl": "compactMjml", "hideSuccessfulItems": true }, "id": "1762259095585", "jobId": "95ac8089-69f3-404e-b902-21d0e878eec2", "jobName": "Backup Job 1", "message": "backup", "scheduleId": "76989b41-8bcf-4438-833a-84ae80125367", "start": 1762259095585, "status": "failure", "infos": [ { "data": { "vms": [ "b25a5709-f1f8-e942-f0cc-f443eb9b9cf3", "e0a3093a-52fd-f8dc-1c39-075eeb9d0314", "f0a673d7-dfb7-5766-86be-6dab83ac264b" ] }, "message": "vms" } ], "tasks": [ { "data": { "type": "VM", "id": "b25a5709-f1f8-e942-f0cc-f443eb9b9cf3", "name_label": "SeedBox" }, "id": "1762259098568", "message": "backup VM", "start": 1762259098568, "status": "failure", "tasks": [ { "id": "1762259098574", "message": "clean-vm", "start": 1762259098574, "status": "success", "end": 1762259098883, "result": { "merge": false } }, { "id": "1762259099293", "message": "snapshot", "start": 1762259099293, "status": "success", "end": 1762259101949, "result": "f46409fb-ebf6-9e8c-a9e9-feb6ed5770db" }, { "data": { "id": "b86b4c88-79f9-4ce8-9fa4-dce78a32ea44", "isFull": true, "type": "remote" }, "id": "1762259101949:0", "message": "export", "start": 1762259101949, "status": "failure", "tasks": [ { "id": "1762259102766", "message": "transfer", "start": 1762259102766, "status": "failure", "end": 1762259790761, "result": { "errno": -122, "code": "Unknown system error -122", "syscall": "write", "message": "Unknown system error -122: Unknown system error -122, write", "name": "Error", "stack": "Error: Unknown system error -122: Unknown system error -122, write" } }, { "id": "1762259791657", "message": "clean-vm", "start": 1762259791657, "status": "success", "end": 1762259791844, "result": { "merge": false } } ], "end": 1762259791845 } ], "infos": [ { "message": "Transfer data using NBD" }, { "message": "will delete snapshot data" }, { "data": { "vdiRef": "OpaqueRef:f4741488-1249-9dd5-8534-08db58d37cea" }, "message": "Snapshot data has been deleted" } ], "end": 1762259791845, "result": { "errno": -2, "code": "ENOENT", "syscall": "open", "path": "/run/xo-server/mounts/b86b4c88-79f9-4ce8-9fa4-dce78a32ea44/xo-vm-backups/b25a5709-f1f8-e942-f0cc-f443eb9b9cf3/vdis/95ac8089-69f3-404e-b902-21d0e878eec2/256af78c-bbf1-429a-b090-bc5a425bca4f/20251104T122502Z.vhd", "message": "ENOENT: no such file or directory, open '/run/xo-server/mounts/b86b4c88-79f9-4ce8-9fa4-dce78a32ea44/xo-vm-backups/b25a5709-f1f8-e942-f0cc-f443eb9b9cf3/vdis/95ac8089-69f3-404e-b902-21d0e878eec2/256af78c-bbf1-429a-b090-bc5a425bca4f/20251104T122502Z.vhd'", "name": "Error", "stack": "Error: ENOENT: no such file or directory, open '/run/xo-server/mounts/b86b4c88-79f9-4ce8-9fa4-dce78a32ea44/xo-vm-backups/b25a5709-f1f8-e942-f0cc-f443eb9b9cf3/vdis/95ac8089-69f3-404e-b902-21d0e878eec2/256af78c-bbf1-429a-b090-bc5a425bca4f/20251104T122502Z.vhd'\nFrom:\n at NfsHandler.addSyncStackTrace (/opt/xo/xo-builds/xen-orchestra-202511031016/@xen-orchestra/fs/dist/local.js:21:26)\n at NfsHandler._openFile (/opt/xo/xo-builds/xen-orchestra-202511031016/@xen-orchestra/fs/dist/local.js:154:35)\n at /opt/xo/xo-builds/xen-orchestra-202511031016/@xen-orchestra/fs/dist/utils.js:29:26\n at new Promise (<anonymous>)\n at NfsHandler.<anonymous> (/opt/xo/xo-builds/xen-orchestra-202511031016/@xen-orchestra/fs/dist/utils.js:24:12)\n at loopResolver (/opt/xo/xo-builds/xen-orchestra-202511031016/node_modules/promise-toolbox/retry.js:83:46)\n at new Promise (<anonymous>)\n at loop (/opt/xo/xo-builds/xen-orchestra-202511031016/node_modules/promise-toolbox/retry.js:85:22)\n at NfsHandler.retry (/opt/xo/xo-builds/xen-orchestra-202511031016/node_modules/promise-toolbox/retry.js:87:10)\n at NfsHandler._openFile (/opt/xo/xo-builds/xen-orchestra-202511031016/node_modules/promise-toolbox/retry.js:103:18)" } }, { "data": { "type": "VM", "id": "e0a3093a-52fd-f8dc-1c39-075eeb9d0314", "name_label": "Docker of Things" }, "id": "1762259098570", "message": "backup VM", "start": 1762259098570, "status": "failure", "tasks": [ { "id": "1762259098574:0", "message": "clean-vm", "start": 1762259098574, "status": "success", "end": 1762259098806, "result": { "merge": false } }, { "id": "1762259099292", "message": "snapshot", "start": 1762259099292, "status": "success", "end": 1762259100729, "result": "ab07b156-96c8-e343-d948-62bb74731c3d" }, { "data": { "id": "b86b4c88-79f9-4ce8-9fa4-dce78a32ea44", "isFull": true, "type": "remote" }, "id": "1762259100729:0", "message": "export", "start": 1762259100729, "status": "failure", "tasks": [ { "id": "1762259102781", "message": "transfer", "start": 1762259102781, "status": "failure", "end": 1762259791458, "result": { "errno": -122, "code": "Unknown system error -122", "syscall": "write", "message": "Unknown system error -122: Unknown system error -122, write", "name": "Error", "stack": "Error: Unknown system error -122: Unknown system error -122, write" } }, { "id": "1762259793276", "message": "clean-vm", "start": 1762259793276, "status": "success", "end": 1762259793447, "result": { "merge": false } } ], "end": 1762259793448 } ], "infos": [ { "message": "Transfer data using NBD" }, { "message": "will delete snapshot data" }, { "data": { "vdiRef": "OpaqueRef:0349ccca-0a6e-460f-e198-a9b17d6d48f3" }, "message": "Snapshot data has been deleted" } ], "end": 1762259793448, "result": { "errno": -2, "code": "ENOENT", "syscall": "open", "path": "/run/xo-server/mounts/b86b4c88-79f9-4ce8-9fa4-dce78a32ea44/xo-vm-backups/e0a3093a-52fd-f8dc-1c39-075eeb9d0314/vdis/95ac8089-69f3-404e-b902-21d0e878eec2/c97068d1-9d82-4d14-bd00-8ebb55aa0933/20251104T122502Z.vhd", "message": "ENOENT: no such file or directory, open '/run/xo-server/mounts/b86b4c88-79f9-4ce8-9fa4-dce78a32ea44/xo-vm-backups/e0a3093a-52fd-f8dc-1c39-075eeb9d0314/vdis/95ac8089-69f3-404e-b902-21d0e878eec2/c97068d1-9d82-4d14-bd00-8ebb55aa0933/20251104T122502Z.vhd'", "name": "Error", "stack": "Error: ENOENT: no such file or directory, open '/run/xo-server/mounts/b86b4c88-79f9-4ce8-9fa4-dce78a32ea44/xo-vm-backups/e0a3093a-52fd-f8dc-1c39-075eeb9d0314/vdis/95ac8089-69f3-404e-b902-21d0e878eec2/c97068d1-9d82-4d14-bd00-8ebb55aa0933/20251104T122502Z.vhd'\nFrom:\n at NfsHandler.addSyncStackTrace (/opt/xo/xo-builds/xen-orchestra-202511031016/@xen-orchestra/fs/dist/local.js:21:26)\n at NfsHandler._openFile (/opt/xo/xo-builds/xen-orchestra-202511031016/@xen-orchestra/fs/dist/local.js:154:35)\n at /opt/xo/xo-builds/xen-orchestra-202511031016/@xen-orchestra/fs/dist/utils.js:29:26\n at new Promise (<anonymous>)\n at NfsHandler.<anonymous> (/opt/xo/xo-builds/xen-orchestra-202511031016/@xen-orchestra/fs/dist/utils.js:24:12)\n at loopResolver (/opt/xo/xo-builds/xen-orchestra-202511031016/node_modules/promise-toolbox/retry.js:83:46)\n at new Promise (<anonymous>)\n at loop (/opt/xo/xo-builds/xen-orchestra-202511031016/node_modules/promise-toolbox/retry.js:85:22)\n at NfsHandler.retry (/opt/xo/xo-builds/xen-orchestra-202511031016/node_modules/promise-toolbox/retry.js:87:10)\n at NfsHandler._openFile (/opt/xo/xo-builds/xen-orchestra-202511031016/node_modules/promise-toolbox/retry.js:103:18)" } }, { "data": { "type": "VM", "id": "f0a673d7-dfb7-5766-86be-6dab83ac264b", "name_label": "UnifiOS" }, "id": "1762259791848", "message": "backup VM", "start": 1762259791848, "status": "success", "tasks": [ { "id": "1762259791852", "message": "clean-vm", "start": 1762259791852, "status": "success", "end": 1762259792007, "result": { "merge": false } }, { "id": "1762259792047", "message": "snapshot", "start": 1762259792047, "status": "success", "end": 1762259794293, "result": "0a91106d-91c1-fbc2-0940-12309a17afa3" }, { "data": { "id": "b86b4c88-79f9-4ce8-9fa4-dce78a32ea44", "isFull": true, "type": "remote" }, "id": "1762259794293:0", "message": "export", "start": 1762259794293, "status": "success", "tasks": [ { "id": "1762259795853", "message": "transfer", "start": 1762259795853, "status": "success", "end": 1762259963650, "result": { "size": 24429723648 } }, { "id": "1762259964823:0", "message": "health check", "start": 1762259964823, "status": "success", "tasks": [ { "id": "1762259964829", "message": "transfer", "start": 1762259964829, "status": "success", "end": 1762260091517, "result": { "size": 0, "id": "d7d4736f-b061-3c13-7257-5f3e15dc40cf" } }, { "id": "1762260091517:0", "message": "vmstart", "start": 1762260091517, "status": "success", "end": 1762260094767 } ], "end": 1762260097234 }, { "id": "1762260097246", "message": "clean-vm", "start": 1762260097246, "status": "success", "warnings": [ { "data": { "path": "/xo-vm-backups/f0a673d7-dfb7-5766-86be-6dab83ac264b/20251104T123635Z.json", "actual": 24429723648, "expected": 24435952128 }, "message": "cleanVm: incorrect backup size in metadata" } ], "end": 1762260097517, "result": { "merge": false } } ], "end": 1762260097518 } ], "infos": [ { "message": "Transfer data using NBD" }, { "message": "will delete snapshot data" }, { "data": { "vdiRef": "OpaqueRef:be3a1abb-7481-e3bb-daff-2533409d1e5d" }, "message": "Snapshot data has been deleted" } ], "end": 1762260097518 } ], "end": 1762260097518 } -
Hi,
@acebmxer said in Backup Jobs failing:
"Error: Unknown system error -122: Unknown system error -122, write"
The one time that I've seen this error was due to the backup remote running out of disk space.
Dan
-
Yep sure enough out of space 5.97TB out of 6TB.... Not sure why i ran out of space. Will look into retention and such.
Thanks.
-
Looking into it looks like VMs were doing full backups everytime when suppose to be delta backups...
Let me know where else i can look to verify, but this is what i see when i look at the backups.
