@florent Backups ran after patch applied... 1 pool had zero issues on backup 100% complete... Other pool while it complete did have some errors... Started a second backup on same pool and giving same results. { "data": { "mode": "delta", "reportWhen": "always", "backupReportTpl": "compactMjml", "hideSuccessfulItems": true }, "id": "1788428437610", "jobId": "c0e31730-7d45-4f3a-8733-8cea444b127d", "jobName": "Salem - Delta Backups", "message": "backup", "proxyId": "7840039e-a3cb-41ed-acee-3690f89a1426", "scheduleId": "ce3c801c-4384-4726-bda1-f6bae788e2f4", "start": 1788428437610, "status": "success", "tasks": [ { "id": "0mtlc477z-gdvpj68kva", "start": 1788428438927, "status": "success", "tasks": [ { "id": "0mtlc478f-guqsurb38qa", "start": 1788428438943, "status": "success", "end": 1788428440120, "result": { "merge": false, "size": 0 }, "message": "clean-vm" }, { "id": "0mtlc48cx-c1y79k93vl", "start": 1788428440401, "status": "success", "end": 1788428442225, "result": "67677eec-494b-6ab6-0447-ac4dba8932a1", "message": "snapshot" }, { "id": "0mtlc49rm-ou0nslm2c7l", "start": 1788428442227, "status": "success", "tasks": [ { "id": "0mtlc724s-62ae1bzdupa", "start": 1788428572300, "status": "success", "end": 1788428932605, "result": { "size": 20385362944 }, "message": "transfer", "data": { "progress": 99 } }, { "id": "0mtlceupa-f476k7iq2u4", "start": 1788428935918, "status": "success", "tasks": [ { "id": "0mtlceutz-yztgqk0sv0c", "start": 1788428936087, "status": "success", "end": 1788429285986, "result": { "size": 20380123136, "id": "5fb5f6f1-709b-b149-4aa6-31514ef2c7f2" }, "message": "transfer" }, { "id": "0mtlcmctg-8z00dmmisds", "start": 1788429285988, "status": "success", "end": 1788429291289, "message": "vmstart" } ], "end": 1788429296722, "message": "health check" }, { "id": "0mtlcml5b-ywe7qa02f7q", "start": 1788429296783, "status": "success", "end": 1788429300723, "result": { "merge": false, "size": 0 }, "message": "clean-vm" } ], "end": 1788429300727, "message": "export", "data": { "id": "19270323-baaa-4a7b-bf1f-e152ae44f263", "isFull": false, "type": "remote" } } ], "warnings": [ { "message": "can't compute delta OpaqueRef:afcf0af9-64c9-845d-207c-789c9c5de78f from OpaqueRef:13ce290f-2194-280e-72c5-9e16271505df, fall back to a full" }, { "message": "can't connect through NBD, fall back to stream export" }, { "message": "Backup fell back to a full" } ], "infos": [ { "message": "will delete snapshot data" }, { "data": { "vdiRef": "OpaqueRef:afcf0af9-64c9-845d-207c-789c9c5de78f" }, "message": "Snapshot data has been deleted" } ], "end": 1788429300728, "message": "backup VM", "data": { "id": "0618e9fe-854e-7f01-1f22-6fb1667806d4", "type": "VM", "name_label": "Salem XO-CE", "progress": 0 } }, { "id": "0mtlc4783-s6m3pnf32pl", "start": 1788428438931, "status": "success", "tasks": [ { "id": "0mtlc478g-epd65nptnrn", "start": 1788428438944, "status": "success", "end": 1788428439927, "result": { "merge": false, "size": 0 }, "message": "clean-vm" }, { "id": "0mtlc486t-mq755lvlxi", "start": 1788428440181, "status": "success", "end": 1788428443614, "result": "d52e8853-1102-8bcd-b588-adbe5b411a6b", "message": "snapshot" }, { "id": "0mtlc4au7-q4va6ntph6c", "start": 1788428443615, "status": "success", "tasks": [ { "id": "0mtlc9pax-xyje1fo25lf", "start": 1788428695641, "status": "success", "end": 1788429704445, "result": { "size": 44032329216 }, "message": "transfer", "data": { "progress": 100 } }, { "id": "0mtlcvdy1-pxmgi1rx7tk", "start": 1788429707353, "status": "success", "tasks": [ { "id": "0mtlcvdzc-uodtj65grrn", "start": 1788429707400, "status": "success", "end": 1788430481457, "result": { "size": 44021317632, "id": "ed4d4ce7-2685-a35a-b1e4-c54a1385364e" }, "message": "transfer" }, { "id": "0mtldbz96-dsndwbb93hv", "start": 1788430481466, "status": "success", "end": 1788430486658, "message": "vmstart" } ], "end": 1788430492516, "message": "health check" }, { "id": "0mtldc7vt-oopf5wcg74e", "start": 1788430492649, "status": "success", "end": 1788430501065, "result": { "merge": false, "size": 0 }, "message": "clean-vm" } ], "end": 1788430501070, "message": "export", "data": { "id": "19270323-baaa-4a7b-bf1f-e152ae44f263", "isFull": false, "type": "remote" } } ], "warnings": [ { "message": "can't compute delta OpaqueRef:82b55a23-b13e-df37-c9f7-11373a42c40b from OpaqueRef:9824e8f1-0199-885c-e69a-d9275cd62d2a, fall back to a full" }, { "message": "can't connect through NBD, fall back to stream export" }, { "message": "Backup fell back to a full" } ], "infos": [ { "message": "will delete snapshot data" }, { "data": { "vdiRef": "OpaqueRef:82b55a23-b13e-df37-c9f7-11373a42c40b" }, "message": "Snapshot data has been deleted" } ], "end": 1788430501071, "message": "backup VM", "data": { "id": "64476a1a-20f5-32a4-6352-47916b2162d8", "type": "VM", "name_label": "Salem UnifiOS", "progress": 0 } }, { "id": "0mtlcmo92-ezud8mmu26", "start": 1788429300806, "status": "success", "tasks": [ { "id": "0mtlcmoaf-ah3ohsdoy7r", "start": 1788429300855, "status": "success", "end": 1788429304315, "result": { "merge": false, "size": 0 }, "message": "clean-vm" }, { "id": "0mtlcmrdk-xm266uwj1ni", "start": 1788429304856, "status": "success", "end": 1788429307794, "result": "dd0d0918-efdd-1a1d-ce94-85afbf0fd25c", "message": "snapshot" }, { "id": "0mtlcmtn7-clidkdhy2ht", "start": 1788429307795, "status": "success", "tasks": [ { "id": "0mtlcs7g8-3k59s9tshcf", "start": 1788429558968, "status": "success", "end": 1788430829213, "result": { "size": 43868711424 }, "message": "transfer", "data": { "progress": 100 } }, { "id": "0mtldjhbz-rfnh54dr3y", "start": 1788430831487, "status": "success", "tasks": [ { "id": "0mtldjhd9-g6rihy05ec7", "start": 1788430831533, "status": "success", "end": 1788431285549, "result": { "size": 43857739776, "id": "e3215b6b-9b5a-0d49-3392-491cc39c19e4" }, "message": "transfer" }, { "id": "0mtldt7ox-r81abwrax4", "start": 1788431285553, "status": "success", "end": 1788431290565, "message": "vmstart" } ], "end": 1788431296314, "message": "health check" }, { "id": "0mtldtg0h-8kvnbox5dg3", "start": 1788431296337, "status": "success", "end": 1788431297610, "result": { "merge": false, "size": 0 }, "message": "clean-vm" } ], "end": 1788431297611, "message": "export", "data": { "id": "19270323-baaa-4a7b-bf1f-e152ae44f263", "isFull": false, "type": "remote" } } ], "warnings": [ { "message": "can't compute delta OpaqueRef:4b3d9930-d795-03f8-c887-f8b7d2a95524 from OpaqueRef:894ead1a-dae6-f156-d8c7-c7f5ba2200ef, fall back to a full" }, { "message": "can't connect through NBD, fall back to stream export" }, { "message": "Backup fell back to a full" } ], "infos": [ { "message": "will delete snapshot data" }, { "data": { "vdiRef": "OpaqueRef:4b3d9930-d795-03f8-c887-f8b7d2a95524" }, "message": "Snapshot data has been deleted" } ], "end": 1788431297611, "message": "backup VM", "data": { "id": "ddf296c8-20a0-6aa9-0d74-ebc029d0d1b3", "type": "VM", "name_label": "Salem UnifiOS", "progress": 0 } } ], "end": 1788431297621, "infos": [ { "data": { "vms": [ "0618e9fe-854e-7f01-1f22-6fb1667806d4", "64476a1a-20f5-32a4-6352-47916b2162d8", "ddf296c8-20a0-6aa9-0d74-ebc029d0d1b3" ] }, "message": "vms" } ] }

@dkidd255 I'm a LOOONG time XCP/XenServer/XCP-ng user in my homelab...And it pains me to say this... But this just works in Proxmox. If you want to help us dig in and troubleshoot/explore this topic...Fine. If you want to actually use your hardware and make progress on your ideas, go to Proxmox. I seem to recall some recent firmware shenanigans from Intel. Intel maybe thought a little too much of their hardware, or they've decided to just scale things back a bit, but newer versions of firmware have reduced VF's available. I don't think the B70 is impacted by this, but it has massive memory in comparison to the smaller cards.

If you are running XOA, not XO from sources, then we could take a look remotely using the support tunnel. Otherwise, make sure that you have patched and rebooted each pool member.

My reported issue is gone with build 7a187 from sourcecode. Day before: tasks 0 id "0mtj9shwq-k2neggnx0sf" properties id "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeca" name "backup VM" type "VM" name_label "unifi-server" progress 0 start 1788303601322 status "success" warnings 0 message "can't compute delta OpaqueRef:aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeee76 from OpaqueRef:aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeee32, fall back to a full" tasks Day after: 1 id "0mtktio1u-lrexahnlve" properties id "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeca" name "backup VM" type "VM" name_label "unifi-server" progress 0 start 1788397201218 status "success" tasks 0

This was sent to me yesterday!! See our Engineering Team lead's response: We evaluated XCP-ng 8.3 some time ago, but its dom0 kernel was too old for our driver to work properly. Running SupremeRAID inside a DomU should be feasible and is similar to the approach we previously proposed for VMware. However, with XCP-ng 8.3, we did not find a practical native path to export the SupremeRAID VD back to dom0 and use it as an XCP-ng SR. Using iSCSI for this purpose would add significant protocol and networking overhead, which is not ideal for high-performance NVMe storage. I revisited XCP-ng 9.0 and the current Xen/XAPI development. A more promising approach is to run SupremeRAID in a dedicated AlmaLinux VM with the GPU and NVMe drives passed through, then use Xen's native xen-blkback interface to export the SupremeRAID block device back to dom0. Dom0 would see the exported VD as a normal Xen block device, which could then potentially be used to create a standard XCP-ng LVM SR. This approach is much more attractive than iSCSI or NVMe/TCP because the data path uses Xen's blkif shared-memory interface rather than a network protocol. xen-blkback itself is an established Xen mechanism, and Xen supports using a separate domain as a block backend. However, XCP-ng does not currently provide complete first-class lifecycle management for this configuration, so we still need to validate the exact behavior on XCP-ng 9.0, particularly persistent attachment, storage VM startup ordering, and recovery after a host or storage VM reboot. If they can help confirm that the GPU and NVMe drives can be passed through to the storage VM, SupremeRAID can run normally there, and the resulting VD can be exported through xen-blkback to dom0 and used as an XCP-ng SR, I think this could be a very solid architecture for SupremeRAID on XCP-ng. As for write durability, SupremeRAID always operates in write-through mode. An I/O is acknowledged only after all associated data, including parity, has been committed to the drives. Therefore, acknowledged writes do not depend on data or parity remaining only in volatile GPU or host memory Can you do the initial testing with SupremeRAID PRO within your environment? I will begin the initial testing and builds with 4 nodes: 1 & 2 are HP DL380 Gen10 - 2x Xeon Gold 6151 36 Cores 384GB RAM 4 3.84Tb PCI4.0 NVMe per node. Twinstore Testing - Run the Build environment on these 3 is a HP Dl360 Gen10 2x Xeon Gold 6151 36 Cores 512Gb RAM 4x1.92TB NVMe Drives SuperServer SYS-122H-TN- X14 2x Xeon 6740 96 cores, 512GB RAM, GRAID Card - 4x Pci5.0 7.68TB drives SupremeRAID. All the nodes have 2x100Gbps ports, 2x25Gbps ports. Arista Backed network.