Our future backup code: test it!
-
@florent Pls advise here when it's fixed, so that I can retry. I will hold back until then.
Backups are essential. -
@manilx the PR is in review, it should be merged in a few hours
thank you for helping us improve the quality
-
@florent Will update and try at the end of today then (GMT)
-
@manilx this is merged
-
-
\o/
-
This post is deleted! -
CR still failed with
Error: HANDLE_INVALID(SR, OpaqueRef:cc116d5f-0a91-b487-b43d-4ebd98495fcd)
Error: Error in the source generator Cannot read properties of undefined (reading 'write')
i found that old CR copies dissapeared from all hosts. -
@Tristis-Oris I still don't have the solution I am working on it .
-
@florent Getting errors again:
{ "data": { "mode": "delta", "reportWhen": "failure" }, "id": "1748134829731", "jobId": "b7215ad7-5616-4620-a6fb-1c5b5b1bd3c6", "jobName": "Mirror Mani", "message": "backup", "scheduleId": "9e8eeb18-65bb-4e12-b1a1-4f7c0490565c", "start": 1748134829731, "status": "failure", "infos": [ { "data": { "vms": [ "0c8fd092-ab36-478e-df85-16b1a01e1ca8", "19242523-e70b-bf39-820d-5e322435012b", "1a65641e-0976-07a8-30f8-3cbd2e4bc63d", "1c596fc8-26a8-7d95-d004-1e694392fc72", "1d7614fc-65f7-4e1c-71f9-da638efd8b4d", "284b64bc-8684-eb3c-b32e-93e455f5854b", "294b867a-e241-fe7a-10b0-aef1de6f693d", "341a6155-908b-66ba-76ee-56a1d5163af1", "43a4d905-7d13-85b8-bed3-f6b805ff26ac", "48ca6a7e-7acf-74fd-4ad9-714315563652", "4d8cad25-b6e5-4d8f-5226-b5908b21a1b6", "52e64134-62e3-9682-4e3f-296a1198db4d", "58af91d1-94cf-cbe5-88cf-560e652741bf", "592e229c-76d7-c0f2-426c-4ef9e80810d0", "5f49e0a4-9397-3da5-e44a-0ab3632cd9f1", "6c9f0275-07bd-a124-e187-47f8389ed93c", "8d573273-49b1-6b84-73a6-42c3346add52", "8e76ba5c-e190-753b-75d6-e67feb1dd85c", "9017c283-51d0-8d22-d13b-328cc9fe4ff7", "903d61e0-c07f-8163-47ae-8e212147a353", "931aaab6-5f7a-d0ef-26a1-1f54063d0941", "9e0c8a72-d0fd-6632-6d57-94224149af05", "9f8088b6-9a5d-0ec5-24d5-118c23100a4b", "a060d911-bf2b-61c5-56bf-11944af25c4a", "a2562d66-00c8-3955-ab43-6a5584044b98", "ac14b70f-9958-7add-f833-64486feba4e7", "b02697d4-23b5-598f-4def-d6b197bc1085", "b5d74e0b-388c-019a-6994-e174c9ca7a51", "bcb4bcfc-c77b-2068-4bec-2826e8ece6be", "c21e8541-c5ea-840e-31aa-095789600230", "c49a5398-38db-4fcd-481b-30181a5503cc", "c5fbe7d1-9f9c-0167-5ed2-1dff1104ef4b", "cc6b00b9-16ec-1990-e44a-503b6e3921c3", "d7964354-aff6-c7e1-0067-52e8e866a965", "ef53c584-f6c2-3aa3-b4c0-802fb20683d9", "f262e69e-f345-bc52-c2dc-60eea7b84c9a", "fab6b6c0-7ce7-91da-fe60-acbcea4ca5be", "fe2dd0ea-f063-9aba-30d5-d2de508b091f" ] }, "message": "vms" } ], "tasks": [ { "data": { "type": "VM", "id": "19242523-e70b-bf39-820d-5e322435012b" }, "id": "1748134830645", "message": "backup VM", "start": 1748134830645, "status": "success", "tasks": [ { "id": "1748134830822", "message": "clean-vm", "start": 1748134830822, "status": "success", "end": 1748134831309, "result": { "merge": false } }, { "data": { "id": "337848e1-b52b-4e08-9bf0-ffce54de4276", "type": "remote" }, "id": "1748134831397", "message": "export", "start": 1748134831397, "status": "success", "tasks": [ { "id": "1748134831413", "message": "transfer", "start": 1748134831413, "status": "success", "end": 1748134937238, "result": { "size": 710934528 } }, { "id": "1748134947620", "message": "clean-vm", "start": 1748134947620, "status": "success", "warnings": [ { "data": { "path": "/xo-vm-backups/19242523-e70b-bf39-820d-5e322435012b/20250524T220611Z.json", "actual": 710934528, "expected": 713224192 }, "message": "cleanVm: incorrect backup size in metadata" } ], "end": 1748134949566, "result": { "merge": true } } ], "end": 1748134965928 } ], "end": 1748134965928 }, { "data": { "type": "VM", "id": "0c8fd092-ab36-478e-df85-16b1a01e1ca8" }, "id": "1748134830649", "message": "backup VM", "start": 1748134830649, "status": "interrupted", "tasks": [ { "id": "1748134830750", "message": "clean-vm", "start": 1748134830750, "status": "success", "warnings": [ { "data": { "path": "/xo-vm-backups/0c8fd092-ab36-478e-df85-16b1a01e1ca8/20250522T161036Z.json", "actual": 71360596992, "expected": 71360597504 }, "message": "cleanVm: incorrect backup size in metadata" } ], "end": 1748134831421, "result": { "merge": false } }, { "data": { "id": "337848e1-b52b-4e08-9bf0-ffce54de4276", "type": "remote" }, "id": "1748134834838", "message": "export", "start": 1748134834838, "status": "success", "tasks": [ { "id": "1748134834899", "message": "transfer", "start": 1748134834899, "status": "success", "end": 1748135312431, "result": { "size": 4857004032 } }, { "id": "1748135312559", "message": "clean-vm", "start": 1748135312559, "status": "success", "warnings": [ { "data": { "parent": "/xo-vm-backups/0c8fd092-ab36-478e-df85-16b1a01e1ca8/vdis/b7215ad7-5616-4620-a6fb-1c5b5b1bd3c6/5cb71c46-ddf5-4058-b29c-3bb436816d45", "child": "/xo-vm-backups/0c8fd092-ab36-478e-df85-16b1a01e1ca8/vdis/b7215ad7-5616-4620-a6fb-1c5b5b1bd3c6/5cb71c46-ddf5-4058-b29c-3bb436816d45/20250524T220111Z.vhd" }, "message": "parent VHD is missing" }, { "data": { "backup": "/xo-vm-backups/0c8fd092-ab36-478e-df85-16b1a01e1ca8/20250524T220111Z.json", "missingVhds": [ "/xo-vm-backups/0c8fd092-ab36-478e-df85-16b1a01e1ca8/vdis/b7215ad7-5616-4620-a6fb-1c5b5b1bd3c6/5cb71c46-ddf5-4058-b29c-3bb436816d45/20250524T220111Z.vhd" ] }, "message": "some VHDs linked to the backup are missing" } ], "end": 1748135312819, "result": { "merge": false } } ], "end": 1748135312823 } ], "warnings": [ { "data": { "attempt": 1, "error": "Expected values to be strictly equal:\n\n0 !== 512\n" }, "message": "Retry the VM mirror backup due to an error" } ] }, { "data": { "type": "VM", "id": "1c596fc8-26a8-7d95-d004-1e694392fc72" }, "id": "1748134966251", "message": "backup VM", "start": 1748134966251, "status": "success", "tasks": [ { "id": "1748134968074", "message": "clean-vm", "start": 1748134968074, "status": "success", "end": 1748134968813, "result": { "merge": false } }, { "data": { "id": "337848e1-b52b-4e08-9bf0-ffce54de4276", "type": "remote" }, "id": "1748134971622", "message": "export", "start": 1748134971622, "status": "success", "tasks": [ { "id": "1748134971997", "message": "transfer", "start": 1748134971997, "status": "success", "end": 1748135312426, "result": { "size": 3261071360 } }, { "id": "1748135314472", "message": "clean-vm", "start": 1748135314472, "status": "success", "warnings": [ { "data": { "path": "/xo-vm-backups/1c596fc8-26a8-7d95-d004-1e694392fc72/20250524T222347Z.json", "actual": 3261071360, "expected": 3264098816 }, "message": "cleanVm: incorrect backup size in metadata" } ], "end": 1748135314652, "result": { "merge": true } } ], "end": 1748135314707 } ], "end": 1748135314707 }, { "data": { "type": "VM", "id": "294b867a-e241-fe7a-10b0-aef1de6f693d" }, "id": "1748135312961", "message": "backup VM", "start": 1748135312961, "status": "interrupted", "tasks": [ { "id": "1748135313037", "message": "clean-vm", "start": 1748135313037, "status": "success", "warnings": [ { "data": { "path": "/xo-vm-backups/294b867a-e241-fe7a-10b0-aef1de6f693d/20250522T161022Z.json", "actual": 11216235520, "expected": 11216236032 }, "message": "cleanVm: incorrect backup size in metadata" } ], "end": 1748135313727, "result": { "merge": false } }, { "data": { "id": "337848e1-b52b-4e08-9bf0-ffce54de4276", "type": "remote" }, "id": "1748135313903", "message": "export", "start": 1748135313903, "status": "failure", "tasks": [ { "id": "1748135313921", "message": "transfer", "start": 1748135313921, "status": "failure", "end": 1748135318829, "result": { "errno": -9, "code": "EBADF", "syscall": "read", "message": "EBADF: bad file descriptor, read", "name": "Error", "stack": "Error: EBADF: bad file descriptor, read\nFrom:\n at NfsHandler.addSyncStackTrace (/opt/xen-orchestra/@xen-orchestra/fs/dist/local.js:21:26)\n at NfsHandler._read (/opt/xen-orchestra/@xen-orchestra/fs/dist/local.js:160:43)\n at NfsHandler.read (/opt/xen-orchestra/@xen-orchestra/fs/dist/abstract.js:259:17)\n at NfsHandler.read (/opt/xen-orchestra/node_modules/limit-concurrency-decorator/index.js:97:24)\n at VhdFile._read (/opt/xen-orchestra/packages/vhd-lib/Vhd/VhdFile.js:126:55)\n at VhdFile.readBlock (/opt/xen-orchestra/packages/vhd-lib/Vhd/VhdFile.js:212:17)\n at RemoteVhd.readBlock (file:///opt/xen-orchestra/@xen-orchestra/backups/disks/RemoteVhd.mjs:144:38)\n at RemoteVhd.buildDiskBlockGenerator (file:///opt/xen-orchestra/@xen-orchestra/disk-transform/dist/Disk.mjs:67:28)" } }, { "id": "1748135318836", "message": "clean-vm", "start": 1748135318836, "status": "success", "end": 1748135318927, "result": { "merge": false } } ], "end": 1748135318948 } ], "warnings": [ { "data": { "attempt": 1, "error": "ENOENT: no such file or directory, open '/run/xo-server/mounts/337848e1-b52b-4e08-9bf0-ffce54de4276/xo-vm-backups/294b867a-e241-fe7a-10b0-aef1de6f693d/vdis/b7215ad7-5616-4620-a6fb-1c5b5b1bd3c6/c87b9871-9ce2-4b39-9fec-9dea2fac3220/20250524T220036Z.vhd'" }, "message": "Retry the VM mirror backup due to an error" } ] }, { "data": { "type": "VM", "id": "43a4d905-7d13-85b8-bed3-f6b805ff26ac" }, "id": "1748135318587", "message": "backup VM", "start": 1748135318587, "status": "success", "tasks": [ { "id": "1748135318860", "message": "clean-vm", "start": 1748135318860, "status": "success", "end": 1748135319739, "result": { "merge": false } }, { "data": { "id": "337848e1-b52b-4e08-9bf0-ffce54de4276", "type": "remote" }, "id": "1748135319979", "message": "export", "start": 1748135319979, "status": "success", "tasks": [ { "id": "1748135319993", "message": "transfer", "start": 1748135319993, "status": "success", "end": 1748135458653, "result": { "size": 870318080 } }, { "id": "1748135473880", "message": "clean-vm", "start": 1748135473880, "status": "success", "warnings": [ { "data": { "path": "/xo-vm-backups/43a4d905-7d13-85b8-bed3-f6b805ff26ac/20250524T220144Z.json", "actual": 870318080, "expected": 872691712 }, "message": "cleanVm: incorrect backup size in metadata" } ], "end": 1748135483581, "result": { "merge": true } } ], "end": 1748135487189 } ], "end": 1748135487190 }, { "data": { "type": "VM", "id": "58af91d1-94cf-cbe5-88cf-560e652741bf" }, "id": "1748135319145", "message": "backup VM", "start": 1748135319145, "status": "success", "tasks": [ { "id": "1748135319356", "message": "clean-vm", "start": 1748135319356, "status": "success", "end": 1748135320011, "result": { "merge": false } }, { "data": { "id": "337848e1-b52b-4e08-9bf0-ffce54de4276", "type": "remote" }, "id": "1748135331775", "message": "export", "start": 1748135331775, "status": "success", "tasks": [ { "id": "1748135331781", "message": "transfer", "start": 1748135331781, "status": "success", "end": 1748135561792, "result": { "size": 2801795072 } }, { "id": "1748135562458", "message": "clean-vm", "start": 1748135562458, "status": "success", "warnings": [ { "data": { "path": "/xo-vm-backups/58af91d1-94cf-cbe5-88cf-560e652741bf/20250524T221027Z.json", "actual": 2801795072, "expected": 2804644352 }, "message": "cleanVm: incorrect backup size in metadata" } ], "end": 1748135562570, "result": { "merge": true } } ], "end": 1748135562664 } ], "end": 1748135562665 }, { "data": { "type": "VM", "id": "6c9f0275-07bd-a124-e187-47f8389ed93c" }, "id": "1748135487646", "message": "backup VM", "start": 1748135487646, "status": "success", "tasks": [ { "id": "1748135488697", "message": "clean-vm", "start": 1748135488697, "status": "success", "end": 1748135499685, "result": { "merge": false } }, { "id": "1748135500529:0", "message": "clean-vm", "start": 1748135500529, "status": "success", "end": 1748135506654, "result": { "merge": false } } ], "infos": [ { "message": "No new data to upload for this VM" } ], "end": 1748135507365 }, { "data": { "type": "VM", "id": "931aaab6-5f7a-d0ef-26a1-1f54063d0941" }, "id": "1748135507681", "message": "backup VM", "start": 1748135507681, "status": "success", "tasks": [ { "id": "1748135509790", "message": "clean-vm", "start": 1748135509790, "status": "success", "end": 1748135517538, "result": { "merge": false } }, { "id": "1748135518256", "message": "clean-vm", "start": 1748135518256, "status": "success", "end": 1748135523041, "result": { "merge": false } } ], "infos": [ { "message": "No new data to upload for this VM" } ], "end": 1748135523539 }, { "data": { "type": "VM", "id": "a060d911-bf2b-61c5-56bf-11944af25c4a" }, "id": "1748135525305", "message": "backup VM", "start": 1748135525305, "status": "success", "tasks": [ { "id": "1748135527194", "message": "clean-vm", "start": 1748135527194, "status": "success", "end": 1748135534840, "result": { "merge": false } }, { "id": "1748135535382", "message": "clean-vm", "start": 1748135535382, "status": "success", "end": 1748135538329, "result": { "merge": false } } ], "infos": [ { "message": "No new data to upload for this VM" } ], "end": 1748135538791 }, { "data": { "type": "VM", "id": "a2562d66-00c8-3955-ab43-6a5584044b98" }, "id": "1748135539187", "message": "backup VM", "start": 1748135539187, "status": "success", "tasks": [ { "id": "1748135561164", "message": "clean-vm", "start": 1748135561164, "status": "success", "end": 1748135562180, "result": { "merge": false } }, { "data": { "id": "337848e1-b52b-4e08-9bf0-ffce54de4276", "type": "remote" }, "id": "1748135562446", "message": "export", "start": 1748135562446, "status": "success", "tasks": [ { "id": "1748135562453", "message": "transfer", "start": 1748135562453, "status": "success", "end": 1748135562527, "result": { "size": 0 } }, { "id": "1748135562801", "message": "clean-vm", "start": 1748135562801, "status": "success", "warnings": [ { "data": { "path": "/xo-vm-backups/a2562d66-00c8-3955-ab43-6a5584044b98/20250524T220635Z.json", "actual": 0, "expected": 134144 }, "message": "cleanVm: incorrect backup size in metadata" } ], "end": 1748135563247, "result": { "merge": true } } ], "end": 1748135563304 } ], "end": 1748135563304 }, { "data": { "type": "VM", "id": "b02697d4-23b5-598f-4def-d6b197bc1085" }, "id": "1748135562737", "message": "backup VM", "start": 1748135562737, "status": "success", "tasks": [ { "id": "1748135562853", "message": "clean-vm", "start": 1748135562853, "status": "success", "warnings": [ { "data": { "path": "/xo-vm-backups/b02697d4-23b5-598f-4def-d6b197bc1085/20250522T161026Z.json", "actual": 11740651520, "expected": 11740652032 }, "message": "cleanVm: incorrect backup size in metadata" } ], "end": 1748135563349, "result": { "merge": false } }, { "data": { "id": "337848e1-b52b-4e08-9bf0-ffce54de4276", "type": "remote" }, "id": "1748135563511", "message": "export", "start": 1748135563511, "status": "success", "tasks": [ { "id": "1748135563531", "message": "transfer", "start": 1748135563531, "status": "success", "end": 1748135898500, "result": { "size": 2713714688 } }, { "id": "1748135910743", "message": "clean-vm", "start": 1748135910743, "status": "success", "warnings": [ { "data": { "path": "/xo-vm-backups/b02697d4-23b5-598f-4def-d6b197bc1085/20250524T220039Z.json", "actual": 2713714688, "expected": 2716501504 }, "message": "cleanVm: incorrect backup size in metadata" } ], "end": 1748135913811, "result": { "merge": true } } ], "end": 1748135915372 } ], "end": 1748135915372 }, { "data": { "type": "VM", "id": "b5d74e0b-388c-019a-6994-e174c9ca7a51" }, "id": "1748135563341", "message": "backup VM", "start": 1748135563341, "status": "success", "tasks": [ { "id": "1748135563363", "message": "clean-vm", "start": 1748135563363, "status": "success", "end": 1748135563857, "result": { "merge": false } }, { "data": { "id": "337848e1-b52b-4e08-9bf0-ffce54de4276", "type": "remote" }, "id": "1748135565987", "message": "export", "start": 1748135565987, "status": "success", "tasks": [ { "id": "1748135565996", "message": "transfer", "start": 1748135565996, "status": "success", "end": 1748136473233, "result": { "size": 13337886720 } }, { "id": "1748136476298", "message": "clean-vm", "start": 1748136476298, "status": "success", "warnings": [ { "data": { "path": "/xo-vm-backups/b5d74e0b-388c-019a-6994-e174c9ca7a51/20250524T220255Z.json", "actual": 13337886720, "expected": 13343341568 }, "message": "cleanVm: incorrect backup size in metadata" } ], "end": 1748136477094, "result": { "merge": true } } ], "end": 1748136477313 } ], "end": 1748136477313 }, { "data": { "type": "VM", "id": "c49a5398-38db-4fcd-481b-30181a5503cc" }, "id": "1748135916054", "message": "backup VM", "start": 1748135916054, "status": "success", "tasks": [ { "id": "1748135920380", "message": "clean-vm", "start": 1748135920380, "status": "success", "end": 1748135930666, "result": { "merge": false } }, { "id": "1748135930919", "message": "clean-vm", "start": 1748135930919, "status": "success", "end": 1748135935805, "result": { "merge": false } } ], "infos": [ { "message": "No new data to upload for this VM" } ], "end": 1748135936610 }, { "data": { "type": "VM", "id": "c5fbe7d1-9f9c-0167-5ed2-1dff1104ef4b" }, "id": "1748135937843", "message": "backup VM", "start": 1748135937843, "status": "success", "tasks": [ { "id": "1748135940204", "message": "clean-vm", "start": 1748135940204, "status": "success", "end": 1748135948613, "result": { "merge": false } }, { "data": { "id": "337848e1-b52b-4e08-9bf0-ffce54de4276", "type": "remote" }, "id": "1748135951349", "message": "export", "start": 1748135951349, "status": "success", "tasks": [ { "id": "1748135951810", "message": "transfer", "start": 1748135951810, "status": "success", "end": 1748136066962, "result": { "size": 960495616 } }, { "id": "1748136083459", "message": "clean-vm", "start": 1748136083459, "status": "success", "warnings": [ { "data": { "path": "/xo-vm-backups/c5fbe7d1-9f9c-0167-5ed2-1dff1104ef4b/20250524T220054Z.json", "actual": 960495616, "expected": 962891264 }, "message": "cleanVm: incorrect backup size in metadata" } ], "end": 1748136093352, "result": { "merge": true } } ], "end": 1748136096139 } ], "end": 1748136096139 } ], "end": 1748136477317, "result": { "errno": -9, "code": "EBADF", "syscall": "scandir", "path": "/run/xo-server/mounts/5dba60dc-c4fc-4d49-9a1e-453522c0d8f7/xo-vm-backups/341a6155-908b-66ba-76ee-56a1d5163af1" } }
