Mirror Incremental backup failure
-
Last night just started getting the following errors.
When I first got error 2 out of 13 VMs failed. When I retired I am down to 1 vm failed. I retried again and same 1 vm fail again.
This is a mirror job to BackBlaze but not sure its related to BackBlaze.
{ "data": { "mode": "delta", "reportWhen": "always", "backupReportTpl": "compactMjml", "hideSuccessfulItems": true }, "id": "1760959398810", "jobId": "de8585b4-d801-44e6-a97a-331725a547c4", "jobName": "Delilah - Mirror Backups Backblaze", "message": "backup", "scheduleId": "1e58ad4a-4b38-49a1-9dfc-4acfa3510681", "start": 1760959398810, "status": "failure", "infos": [ { "data": { "vms": [ "0dc3c884-97bb-bcb5-62d1-50ddfebf1519", "102a5ac2-ab7f-68ca-9fa1-b549d64ec4a9", "291f9d05-88c8-fc37-107a-368a55c649be", "2c21c005-9230-f3f3-cdf6-2d39407b3456", "342b470e-fac9-6579-f82d-2517b8de6112", "4567b8da-3a3c-00a4-cf6b-ccdae302a645", "5703adef-d804-6b15-ba2f-7b3357a711bb", "5a2c18ca-e9ce-abfc-8bd7-2794f94c6c3a", "a8858f86-ceab-2d5d-5d46-1b083c3fa399", "a89ed090-610a-8f41-58b1-1cb1c8d6517a", "ab9e8cde-fad8-5089-29c8-c904a810857a", "b0c21701-7309-dd9c-fa2a-bdc84a24565e", "bcc182c6-dbb7-33c4-3e8b-c7db9b246e7b", "c9dadc90-6fb8-eee7-d3e5-e1ae1024a77f", "d2025b59-6268-6bb9-28cd-e4812395563c", "d4352870-d73c-5cda-9470-ffd7cd01e251", "f96015ff-6a52-aec0-4581-4a65029437a7" ] }, "message": "vms" } ], "tasks": [ { "data": { "type": "VM", "id": "102a5ac2-ab7f-68ca-9fa1-b549d64ec4a9" }, "id": "1760959399518", "message": "backup VM", "start": 1760959399518, "status": "success", "tasks": [ { "id": "1760959399540", "message": "clean-vm", "start": 1760959399540, "status": "success", "end": 1760959448830, "result": { "merge": false } }, { "id": "1760959448856:1", "message": "clean-vm", "start": 1760959448856, "status": "success", "end": 1760959493080, "result": { "merge": false } } ], "infos": [ { "message": "No new data to upload for this VM" }, { "message": "No healthCheck needed because no data was transferred." } ], "end": 1760959493080 }, { "data": { "type": "VM", "id": "291f9d05-88c8-fc37-107a-368a55c649be" }, "id": "1760959399538", "message": "backup VM", "start": 1760959399538, "status": "success", "tasks": [ { "id": "1760959399660", "message": "clean-vm", "start": 1760959399660, "status": "success", "end": 1760959435933, "result": { "merge": false } }, { "id": "1760959436036", "message": "clean-vm", "start": 1760959436036, "status": "success", "end": 1760959473882, "result": { "merge": false } } ], "infos": [ { "message": "No new data to upload for this VM" }, { "message": "No healthCheck needed because no data was transferred." } ], "end": 1760959473882 }, { "data": { "type": "VM", "id": "2c21c005-9230-f3f3-cdf6-2d39407b3456" }, "id": "1760959473911", "message": "backup VM", "start": 1760959473911, "status": "success", "tasks": [ { "id": "1760959473929", "message": "clean-vm", "start": 1760959473929, "status": "success", "end": 1760959509274, "result": { "merge": false } }, { "id": "1760959509392", "message": "clean-vm", "start": 1760959509392, "status": "success", "end": 1760959545428, "result": { "merge": false } } ], "infos": [ { "message": "No new data to upload for this VM" }, { "message": "No healthCheck needed because no data was transferred." } ], "end": 1760959545428 }, { "data": { "type": "VM", "id": "342b470e-fac9-6579-f82d-2517b8de6112" }, "id": "1760959493083", "message": "backup VM", "start": 1760959493083, "status": "success", "tasks": [ { "id": "1760959493100", "message": "clean-vm", "start": 1760959493100, "status": "success", "end": 1760959516729, "result": { "merge": false } }, { "id": "1760959516758", "message": "clean-vm", "start": 1760959516758, "status": "success", "end": 1760959541506, "result": { "merge": false } } ], "infos": [ { "message": "No new data to upload for this VM" }, { "message": "No healthCheck needed because no data was transferred." } ], "end": 1760959541506 }, { "data": { "type": "VM", "id": "4567b8da-3a3c-00a4-cf6b-ccdae302a645" }, "id": "1760959541521", "message": "backup VM", "start": 1760959541521, "status": "success", "tasks": [ { "id": "1760959541558", "message": "clean-vm", "start": 1760959541558, "status": "success", "end": 1760959549245, "result": { "merge": false } }, { "id": "1760959549340", "message": "clean-vm", "start": 1760959549340, "status": "success", "end": 1760959555142, "result": { "merge": false } } ], "infos": [ { "message": "No new data to upload for this VM" }, { "message": "No healthCheck needed because no data was transferred." } ], "end": 1760959555142 }, { "data": { "type": "VM", "id": "5703adef-d804-6b15-ba2f-7b3357a711bb" }, "id": "1760959545440", "message": "backup VM", "start": 1760959545440, "status": "success", "tasks": [ { "id": "1760959545457", "message": "clean-vm", "start": 1760959545457, "status": "success", "end": 1760959545948, "result": { "merge": false } }, { "id": "1760959545981:1", "message": "clean-vm", "start": 1760959545981, "status": "success", "end": 1760959546221, "result": { "merge": false } } ], "infos": [ { "message": "This VM is excluded by the job filter" }, { "message": "No healthCheck needed because no data was transferred." } ], "end": 1760959546222 }, { "data": { "type": "VM", "id": "5a2c18ca-e9ce-abfc-8bd7-2794f94c6c3a" }, "id": "1760959546230", "message": "backup VM", "start": 1760959546230, "status": "success", "tasks": [ { "id": "1760959546246", "message": "clean-vm", "start": 1760959546246, "status": "success", "end": 1760959564273, "result": { "merge": false } }, { "id": "1760959564357:1", "message": "clean-vm", "start": 1760959564357, "status": "success", "end": 1760959578416, "result": { "merge": false } } ], "infos": [ { "message": "This VM is excluded by the job filter" }, { "message": "No healthCheck needed because no data was transferred." } ], "end": 1760959578416 }, { "data": { "type": "VM", "id": "a8858f86-ceab-2d5d-5d46-1b083c3fa399" }, "id": "1760959555146", "message": "backup VM", "start": 1760959555146, "status": "success", "tasks": [ { "id": "1760959555178", "message": "clean-vm", "start": 1760959555178, "status": "success", "end": 1760959629767, "result": { "merge": false } }, { "id": "1760959629795", "message": "clean-vm", "start": 1760959629795, "status": "success", "end": 1760959692700, "result": { "merge": false } } ], "infos": [ { "message": "No new data to upload for this VM" }, { "message": "No healthCheck needed because no data was transferred." } ], "end": 1760959692700 }, { "data": { "type": "VM", "id": "ab9e8cde-fad8-5089-29c8-c904a810857a" }, "id": "1760959578446", "message": "backup VM", "start": 1760959578446, "status": "success", "tasks": [ { "id": "1760959578466", "message": "clean-vm", "start": 1760959578466, "status": "success", "end": 1760959632497, "result": { "merge": false } }, { "id": "1760959632590:0", "message": "clean-vm", "start": 1760959632590, "status": "success", "end": 1760959696721, "result": { "merge": false } } ], "infos": [ { "message": "No new data to upload for this VM" }, { "message": "No healthCheck needed because no data was transferred." } ], "end": 1760959696721 }, { "data": { "type": "VM", "id": "b0c21701-7309-dd9c-fa2a-bdc84a24565e" }, "id": "1760959692705", "message": "backup VM", "start": 1760959692705, "status": "success", "tasks": [ { "id": "1760959692720", "message": "clean-vm", "start": 1760959692720, "status": "success", "end": 1760959743473, "result": { "merge": false } }, { "id": "1760959743495:1", "message": "clean-vm", "start": 1760959743495, "status": "success", "end": 1760959795554, "result": { "merge": false } } ], "infos": [ { "message": "This VM is excluded by the job filter" }, { "message": "No healthCheck needed because no data was transferred." } ], "end": 1760959795554 }, { "data": { "type": "VM", "id": "bcc182c6-dbb7-33c4-3e8b-c7db9b246e7b" }, "id": "1760959696739", "message": "backup VM", "start": 1760959696739, "status": "success", "tasks": [ { "id": "1760959696765", "message": "clean-vm", "start": 1760959696765, "status": "success", "end": 1760959726538, "result": { "merge": false } }, { "id": "1760959726636:0", "message": "clean-vm", "start": 1760959726636, "status": "success", "end": 1760959754694, "result": { "merge": false } } ], "infos": [ { "message": "No new data to upload for this VM" }, { "message": "No healthCheck needed because no data was transferred." } ], "end": 1760959754694 }, { "data": { "type": "VM", "id": "c9dadc90-6fb8-eee7-d3e5-e1ae1024a77f" }, "id": "1760959754698", "message": "backup VM", "start": 1760959754698, "status": "failure", "tasks": [ { "id": "1760959754715", "message": "clean-vm", "start": 1760959754715, "status": "failure", "tasks": [ { "id": "1760959795421", "message": "merge", "start": 1760959795421, "status": "failure", "end": 1760959824455, "result": { "name": "InvalidRequest", "$fault": "client", "$metadata": { "httpStatusCode": 400, "requestId": "4f8fb6bb5ee77752", "extendedRequestId": "aYvU1qDBHYcE2jmTNOBg5bGUZZSo0UTA/", "attempts": 1, "totalRetryDelay": 0 }, "Code": "InvalidRequest", "message": "no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532", "stack": "InvalidRequest: no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532\n at throwDefaultError (/usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:836:20)\n at /usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:845:5\n at de_CommandError (/usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/client-s3/dist-cjs/index.js:4739:14)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-serde/dist-cjs/index.js:35:20\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:485:18\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-retry/dist-cjs/index.js:320:38\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:110:22\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:138:14\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-logger/dist-cjs/index.js:34:22" } } ], "end": 1760959824455, "result": { "name": "InvalidRequest", "$fault": "client", "$metadata": { "httpStatusCode": 400, "requestId": "4f8fb6bb5ee77752", "extendedRequestId": "aYvU1qDBHYcE2jmTNOBg5bGUZZSo0UTA/", "attempts": 1, "totalRetryDelay": 0 }, "Code": "InvalidRequest", "message": "no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532", "stack": "InvalidRequest: no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532\n at throwDefaultError (/usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:836:20)\n at /usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:845:5\n at de_CommandError (/usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/client-s3/dist-cjs/index.js:4739:14)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-serde/dist-cjs/index.js:35:20\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:485:18\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-retry/dist-cjs/index.js:320:38\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:110:22\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:138:14\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-logger/dist-cjs/index.js:34:22" } }, { "id": "1760959824478:1", "message": "clean-vm", "start": 1760959824478, "status": "failure", "tasks": [ { "id": "1760959861600", "message": "merge", "start": 1760959861600, "status": "failure", "end": 1760959889737, "result": { "name": "InvalidRequest", "$fault": "client", "$metadata": { "httpStatusCode": 400, "requestId": "130a17c8cff103db", "extendedRequestId": "aYpo1YjBJYQo2X2RMOP85oWUXZUo09DDB", "attempts": 1, "totalRetryDelay": 0 }, "Code": "InvalidRequest", "message": "no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532", "stack": "InvalidRequest: no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532\n at throwDefaultError (/usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:836:20)\n at /usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:845:5\n at de_CommandError (/usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/client-s3/dist-cjs/index.js:4739:14)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-serde/dist-cjs/index.js:35:20\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:485:18\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-retry/dist-cjs/index.js:320:38\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:110:22\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:138:14\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-logger/dist-cjs/index.js:34:22" } } ], "end": 1760959889737, "result": { "name": "InvalidRequest", "$fault": "client", "$metadata": { "httpStatusCode": 400, "requestId": "130a17c8cff103db", "extendedRequestId": "aYpo1YjBJYQo2X2RMOP85oWUXZUo09DDB", "attempts": 1, "totalRetryDelay": 0 }, "Code": "InvalidRequest", "message": "no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532", "stack": "InvalidRequest: no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532\n at throwDefaultError (/usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:836:20)\n at /usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:845:5\n at de_CommandError (/usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/client-s3/dist-cjs/index.js:4739:14)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-serde/dist-cjs/index.js:35:20\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:485:18\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-retry/dist-cjs/index.js:320:38\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:110:22\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:138:14\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-logger/dist-cjs/index.js:34:22" } } ], "infos": [ { "message": "No new data to upload for this VM" }, { "message": "No healthCheck needed because no data was transferred." } ], "end": 1760959889738 }, { "data": { "type": "VM", "id": "d4352870-d73c-5cda-9470-ffd7cd01e251" }, "id": "1760959795559", "message": "backup VM", "start": 1760959795559, "status": "success", "tasks": [ { "id": "1760959795574", "message": "clean-vm", "start": 1760959795574, "status": "success", "end": 1760959804514, "result": { "merge": false } }, { "id": "1760959804585", "message": "clean-vm", "start": 1760959804585, "status": "success", "end": 1760959810828, "result": { "merge": false } } ], "infos": [ { "message": "No new data to upload for this VM" }, { "message": "No healthCheck needed because no data was transferred." } ], "end": 1760959810828 } ], "end": 1760959889738 }Edit - posted correct error log.